Архитектура платформы

 

Screenshots Insights.online.fa.ru

 

Данные о городе проживания из профиля

Пол из профиля на online.af.ru 

Регистрация по дням

Список курсов из online.fa.ru

Insights.online.fa.ru

task: LastCountryOfUser

Географическая метка поьзователей 

remote-task --host insights --user vladimir --private-key /home/vladimir/yandex-fa.pem --remote-name analyticstack --skip-setup --wait  --local-scheduler   \

LastCountryOfUser   --interval 2022-01-01-2022-12-12 \

  --n-reduce-tasks 1 \

  --overwrite-n-days 365

task: ModuleEngagementWorkflowTask

 remote-task --host insights --user vladimir --private-key /home/vladimir/yandex-fa.pem --local-scheduler --remote-name analyticstack --skip-setup --wait ModuleEngagementWorkflowTask --date $(date +%Y-%m-%d -d "2021-12-12") --indexing-tasks 5 --throttle 0.5 --n-reduce-tasks 1

--local-scheduler - указатель использовать локальный луиджи

hadoop task: ImportEnrollmentsIntoMysql

 remote-task --host insights --user vladimir --private-key /home/vladimir/yandex-fa.pem --remote-name analyticstack --skip-setup --wait ImportEnrollmentsIntoMysql --local-scheduler \

  --interval 2018-01-01-2018-12-12 \

  --n-reduce-tasks 1 \

  --overwrite-mysql \

  --overwrite-hive  --overwrite-n-days 365

host - имя машины которую анализируем

private-key - ключ доступа по ssh

 analytics.tasks

    # common

    sqoop-import = edx.analytics.tasks.common.sqoop:SqoopImportFromMysql

    insert-into-table = edx.analytics.tasks.common.mysql_load:MysqlInsertTask

    bigquery-load = edx.analytics.tasks.common.bigquery_load:BigQueryLoadTask

    # insights

    answer-dist = edx.analytics.tasks.insights.answer_dist:AnswerDistributionPerCourse

    calendar = edx.analytics.tasks.insights.calendar_task:CalendarTableTask

    course_blocks = edx.analytics.tasks.insights.course_blocks:CourseBlocksApiDataTask

    course_list = edx.analytics.tasks.insights.course_list:CourseListApiDataTask

    database-import = edx.analytics.tasks.insights.database_imports:ImportAllDatabaseTablesTask

    engagement = edx.analytics.tasks.insights.module_engagement:ModuleEngagementDataTask

    enrollments = edx.analytics.tasks.insights.enrollments:ImportEnrollmentsIntoMysql

    location-per-course = edx.analytics.tasks.insights.location_per_course:LastCountryOfUser

    problem_response = edx.analytics.tasks.insights.problem_response:LatestProblemResponseDataTask

    tags-dist = edx.analytics.tasks.insights.tags_dist:TagsDistributionPerCourse

    user-activity = edx.analytics.tasks.insights.user_activity:InsertToMysqlCourseActivityTask

    video = edx.analytics.tasks.insights.video:InsertToMysqlAllVideoTask

    # data_api

    grade-dist = edx.analytics.tasks.data_api.studentmodule_dist:GradeDistFromSqoopToMySQLWorkflow

    student_engagement = edx.analytics.tasks.data_api.student_engagement:StudentEngagementTask

    # warehouse:

    event-type-dist = edx.analytics.tasks.warehouse.event_type_dist:PushToVerticaEventTypeDistributionTask

    load-course-catalog = edx.analytics.tasks.warehouse.load_internal_reporting_course_catalog:PullDiscoveryCoursesAPIData

    load-d-certificates = edx.analytics.tasks.warehouse.load_internal_reporting_certificates:LoadInternalReportingCertificatesToWarehouse

    load-d-country = edx.analytics.tasks.warehouse.load_internal_reporting_country:LoadInternalReportingCountryToWarehouse

    load-d-user = edx.analytics.tasks.warehouse.load_internal_reporting_user:LoadInternalReportingUserToWarehouse

    load-d-user-course = edx.analytics.tasks.warehouse.load_internal_reporting_user_course:LoadUserCourseSummary

    load-events = edx.analytics.tasks.warehouse.load_internal_reporting_events:TrackingEventRecordDataTask

    load-f-user-activity = edx.analytics.tasks.warehouse.load_internal_reporting_user_activity:LoadInternalReportingUserActivityToWarehouse

    load-internal-database = edx.analytics.tasks.warehouse.load_internal_reporting_database:ImportMysqlToVerticaTask

    load-internal-active-users = edx.analytics.tasks.warehouse.load_internal_reporting_active_users:LoadInternalReportingActiveUsersToWarehouse

    load-warehouse = edx.analytics.tasks.warehouse.load_warehouse:LoadWarehouseWorkflow

    load-warehouse-bigquery=edx.analytics.tasks.warehouse.load_warehouse_bigquery:LoadWarehouseBigQueryTask

    push_to_vertica_lms_courseware_link_clicked = edx.analytics.tasks.warehouse.lms_courseware_link_clicked:PushToVerticaLMSCoursewareLinkClickedTask

    run-vertica-sql-script = edx.analytics.tasks.warehouse.run_vertica_sql_script:RunVerticaSqlScriptTask

    run-vertica-sql-scripts = edx.analytics.tasks.warehouse.run_vertica_sql_scripts:RunVerticaSqlScriptTask

    test-vertica-sqoop = edx.analytics.tasks.common.vertica_export:VerticaSchemaToBigQueryTask

    # financial:

    cybersource = edx.analytics.tasks.warehouse.financial.cybersource:DailyPullFromCybersourceTask

    ed_services_report = edx.analytics.tasks.warehouse.financial.ed_services_financial_report:BuildEdServicesReportTask

    financial_reports  = edx.analytics.tasks.warehouse.financial.finance_reports:BuildFinancialReportsTask

    orders = edx.analytics.tasks.warehouse.financial.orders_import:OrderTableTask

    payment_reconcile = edx.analytics.tasks.warehouse.financial.reconcile:ReconcileOrdersAndTransactionsTask

    paypal = edx.analytics.tasks.warehouse.financial.paypal:PaypalTransactionsByDayTask

    # export:

    data_obfuscation   = edx.analytics.tasks.export.data_obfuscation:ObfuscatedCourseDumpTask

    dump-student-module = edx.analytics.tasks.export.database_exports:StudentModulePerCourseTask

    events_obfuscation = edx.analytics.tasks.export.events_obfuscation:ObfuscateCourseEventsTask

    export-events = edx.analytics.tasks.export.event_exports:EventExportTask

    export-events-by-course = edx.analytics.tasks.export.event_exports_by_course:EventExportByCourseTask

    export-student-module = edx.analytics.tasks.export.database_exports:StudentModulePerCourseAfterImportWorkflow

    obfuscation = edx.analytics.tasks.export.obfuscation:ObfuscatedCourseTask

    # monitor:

    all_events_report = edx.analytics.tasks.monitor.total_events_report:TotalEventsReportWorkflow

    enrollment_validation = edx.analytics.tasks.monitor.enrollment_validation:CourseEnrollmentValidationTask

    overall_events = edx.analytics.tasks.monitor.overall_events:TotalEventsDailyTask

    noop = edx.analytics.tasks.monitor.performance:ParseEventLogPerformanceTask

    # enterprise:

    enterprise_enrollments = edx.analytics.tasks.enterprise.enterprise_enrollments:ImportEnterpriseEnrollmentsIntoMysql

mapreduce.engine =

    hadoop = edx.analytics.tasks.common.mapreduce:MapReduceJobRunner

    local = luigi.contrib.hadoop:LocalJobRunner

    emu = edx.analytics.tasks.common.mapreduce:EmulatedMapReduceJobRunner

Staff Graded Assignment XBlock

 Дописан функционал создания zip файлов присланных студентами файлов в директории
course.name - course.run - block_id

Пример:

/edx/var/edxapp/media/fa/finadvice/edx_sga_zip/2022/9a0ceeb19/Ivanov.zip

Создание zip запускается так:

./manage.py lms --setting=aws sga_migrate_submissions

--course 'course-v1:fa+finadvice+2022_9_1'

--block 'block-v1:fa+finadvice+2022_9_1+type@edx_sga+block@0a1af775926a41b798b841209af15c75'

Код: кинуть в папку management 

sga_migrate_submissions.py

import json

from django.core.management.base import BaseCommand, CommandError # lint-amnesty, pylint: disable=import-error

from courseware.courses import get_course_by_id # lint-amnesty, pylint: disable=import-error

from courseware.models import StudentModule # lint-amnesty, pylint: disable=import-error

from opaque_keys.edx.keys import CourseKey # lint-amnesty, pylint: disable=import-error

from student.models import anonymous_id_for_user # lint-amnesty, pylint: disable=import-error

from submissions import api as submissions_api # lint-amnesty, pylint: disable=import-error

from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=import-error

from io import BytesIO

import zipfile

import os

import shutil

from opaque_keys.edx.locator import BlockUsageLocator

from django.core.files.storage import default_storage

from django.core.files.base import ContentFile

from edx_sga.constants import BLOCK_SIZE, ITEM_TYPE

from edx_sga.tasks import (

get_zip_file_name,

get_zip_file_path,

get_zip_file_dir,

zip_student_submissions,

_get_student_submissions

)

class Command(BaseCommand):

"""

Migrates existing SGA submissions for a course from old SGA implementation

to newer version that uses the 'submissions' application.

sga_migrate_submissions --course 'course-v1:fa+finadvice+2022_1' --block 'block-v1:fa+finadvice+2022_1+type@edx_sga+block@0a1af775926a41b798b841209af15c75'

"""

# args = ["<course_id>","<block_id>"]

def add_arguments(self, parser):

parser.add_argument(

'-c',

'--course',

metavar='COURSE_ID',

dest='course',

default=False,

help='zip course'

)

parser.add_argument(

'-b',

'--block',

metavar='BLOCK_ID',

dest='block',

default=False,

help='zip block'

)

def handle(self, *args, **options):

"""

Migrates existing SGA submissions.

"""

course_id = options['course']

block_id = options['block']

# course_id = 'course-v1:fa+finadvice+2022_9_1'

course_key = CourseKey.from_string(course_id)

course = get_course_by_id(course_key)

# block_id = 'block-v1:fa+finadvice+2022_9_1+type@edx_sga+block@f7324e8b3f3f464e9d532359a0ceeb19'

locator_unicode = block_id

locator = BlockUsageLocator.from_string(locator_unicode)

directory = get_zip_file_dir(locator)

directory_abs = os.path.join('/edx/var/edxapp/media', directory)

direcory_with_run=os.path.join(directory_abs, locator.run)

directory_with_block_id=os.path.join(direcory_with_run, locator.block_id)

if not os.path.exists(direcory_with_run):

os.makedirs(direcory_with_run)

if not os.path.exists(directory_with_block_id):

os.makedirs(directory_with_block_id)

student_submissions = _get_student_submissions(block_id, course_id, locator)

zip_file_bytes = BytesIO()

with zipfile.ZipFile(zip_file_bytes, 'w') as zip_pointer:

for student_username, submission_file_path in student_submissions:

with default_storage.open(submission_file_path, 'rb') as destination_file:

filename_in_zip = '{}_{}'.format(student_username,os.path.basename(submission_file_path))

print filename_in_zip

zip_pointer.writestr(filename_in_zip, destination_file.read())

zip_file_bytes.seek(0)

filename_with_block_id=os.path.join(directory_with_block_id, filename_in_zip +'.zip' )

with open(filename_with_block_id, "wb") as zip_file_pointer:

zip_file_pointer.write(zip_file_bytes.getvalue())

print 'created '+filename_with_block_id

Hadoop + hive + Spark

Добавляем файл для дальнейшего анализа  

 /edx/app/hadoop/hadoop/bin/hdfs dfs -put -f /edx/var/log/tracking/tracking.log hdfs://localhost:9000/data/tracking.log

Проверяем

hdfs dfs -ls /data

Запускаем расчёт через подключение к удаленному хосту по ключу

remote-task --host server3 --user vladimir --private-key /home/vladimir/yandex-fa.pem --remote-name analyticstack --skip-setup --wait \ 

AnswerDistributionWorkflow --local-scheduler \

--src hdfs://localhost:9000/data \

--dest  hdfs://localhost:9000/edx-analytics-pipeline \

--name hadoop  --output-root hdfs://localhost:9000/output/ \

--include 'tracking.log.gz'  \

--manifest hdfs://localhost:9000/data/manifest.txt  \

--base-input-format "org.edx.hadoop.input.ManifestTextInputFormat" \

 --lib-jar "hdfs://localhost:9000/edx-analytics-pipeline/packages/edx-analytics-hadoop-util.jar" --n-reduce-tasks 1 \

--marker hdfs://localhost:9000/edx-analytics-pipeline/marker/  \

--credentials "/edx/etc/edx-analytics-pipeline/output.json" 

 

Верификация по фото

Как продлить действие верификации до 2025

SQL: 

Update edxapp.verify_student_softwaresecurephotoverification SET submitted_at='2024-12-12 12:12:12.000000', updated_at='2022-11-18 12:12:12.808484' WHERE edxapp.verify_student_softwaresecurephotoverification.submitted_at is not null;

Проверить настройки

 

Почта

 Настройки почты 

    "EMAIL_HOST_PASSWORD": "**********,

    "EMAIL_HOST_USER": "vvandropov@fa.ru",

"DEFAULT_FROM_EMAIL": "vvandropov@fa.ru",

    "EMAIL_BACKEND": "django_smtp_ssl.SSLEmailBackend",

    "EMAIL_HOST": "smtp.mail.ru",

    "EMAIL_PORT": 465,

    "EMAIL_USE_TLS": false,

    "EMAIL_USE_SSL": true,

Настройки  postfix

mydestination = edx3.ru-central1.internal, localhost

relayhost =[smtp.mail.ru]:465

alias_database = hash:/etc/aliases

alias_maps = hash:/etc/aliases

command_directory = /usr/sbin

data_directory = /var/lib/postfix

debug_peer_level = 2

debugger_command = 'PATH=/bin:/usr/bin:/usr/local/bin:/usr/X11R6/bin ddd $daemon_directory/$process_name $process_id & sleep 5'

html_directory = no

inet_interfaces = localhost

inet_protocols = ipv4

mail_owner = postfix

manpage_directory = /usr/share/man

mydomain = edx3.ru-central1.internal

myhostname = edx3.ru-central1.internal

mynetworks = 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16

queue_directory = /var/spool/postfix

setgid_group = postdrop

smtp_tls_security_level = encrypt

smtp_use_tls = yes

unknown_local_recipient_reject_code = 550

sendmail_path = /usr/sbin/sendmail

smtp_tls_CAfile = /etc/ssl/certs/ca-certificates.crt

mailq_path = /usr/bin/mailq

newaliases_path = /usr/bin/newaliases

smtp_generic_maps = hash:/etc/postfix/generic

smtp_sasl_auth_enable = yes

smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd

smtp_sasl_security_options = noanonymous

smtp_tls_wrappermode = yes

compatibility_level = 2

aliase 

root;  vvandropov@fa.ru

generic 

root  mycoolcamera@mail.r

sasl-password 

[smtp.mail.ru]:465 mycoolcamera@mail.ru:*********

commands after

postmap /etc/postfix/sasl_passwd

root@server3:/home/vladimir# postmap /etc/postfix/generic

root@server3:/home/vladimir# systemctl restart postfix 

root@server3:/home/vladimir# postfix reload

root@server3:/home/vladimir# systemctl restart postfix 

root@server3:/home/vladimir# echo "hllo world" |mail -s "tm"  mycoolcamera@mail.ru

root@server3:/home/vladimir# tail -111 /var/log/mail.log