пятница, 13 января 2023 г.

Screenshots Insights.online.fa.ru

 

Данные о городе проживания из профиля

Пол из профиля на online.af.ru 



Регистрация по дням

Список курсов из online.fa.ru

Insights.online.fa.ru

пятница, 23 декабря 2022 г.

task: LastCountryOfUser

Географическая метка поьзователей 

remote-task --host insights --user vladimir --private-key /home/vladimir/yandex-fa.pem --remote-name analyticstack --skip-setup --wait  --local-scheduler   \

LastCountryOfUser   --interval 2022-01-01-2022-12-12 \

  --n-reduce-tasks 1 \

  --overwrite-n-days 365

четверг, 22 декабря 2022 г.

task: ModuleEngagementWorkflowTask

 remote-task --host insights --user vladimir --private-key /home/vladimir/yandex-fa.pem --local-scheduler --remote-name analyticstack --skip-setup --wait ModuleEngagementWorkflowTask --date $(date +%Y-%m-%d -d "2021-12-12") --indexing-tasks 5 --throttle 0.5 --n-reduce-tasks 1


--local-scheduler - указатель использовать локальный луиджи


hadoop task: ImportEnrollmentsIntoMysql

 remote-task --host insights --user vladimir --private-key /home/vladimir/yandex-fa.pem --remote-name analyticstack --skip-setup --wait ImportEnrollmentsIntoMysql --local-scheduler \

  --interval 2018-01-01-2018-12-12 \

  --n-reduce-tasks 1 \

  --overwrite-mysql \

  --overwrite-hive  --overwrite-n-days 365


host - имя машины которую анализируем

private-key - ключ доступа по ssh

четверг, 1 декабря 2022 г.

analytics.tasks

 analytics.tasks


    # common

    sqoop-import = edx.analytics.tasks.common.sqoop:SqoopImportFromMysql

    insert-into-table = edx.analytics.tasks.common.mysql_load:MysqlInsertTask

    bigquery-load = edx.analytics.tasks.common.bigquery_load:BigQueryLoadTask


    # insights

    answer-dist = edx.analytics.tasks.insights.answer_dist:AnswerDistributionPerCourse

    calendar = edx.analytics.tasks.insights.calendar_task:CalendarTableTask

    course_blocks = edx.analytics.tasks.insights.course_blocks:CourseBlocksApiDataTask

    course_list = edx.analytics.tasks.insights.course_list:CourseListApiDataTask

    database-import = edx.analytics.tasks.insights.database_imports:ImportAllDatabaseTablesTask

    engagement = edx.analytics.tasks.insights.module_engagement:ModuleEngagementDataTask

    enrollments = edx.analytics.tasks.insights.enrollments:ImportEnrollmentsIntoMysql

    location-per-course = edx.analytics.tasks.insights.location_per_course:LastCountryOfUser

    problem_response = edx.analytics.tasks.insights.problem_response:LatestProblemResponseDataTask

    tags-dist = edx.analytics.tasks.insights.tags_dist:TagsDistributionPerCourse

    user-activity = edx.analytics.tasks.insights.user_activity:InsertToMysqlCourseActivityTask

    video = edx.analytics.tasks.insights.video:InsertToMysqlAllVideoTask


    # data_api

    grade-dist = edx.analytics.tasks.data_api.studentmodule_dist:GradeDistFromSqoopToMySQLWorkflow

    student_engagement = edx.analytics.tasks.data_api.student_engagement:StudentEngagementTask


    # warehouse:

    event-type-dist = edx.analytics.tasks.warehouse.event_type_dist:PushToVerticaEventTypeDistributionTask

    load-course-catalog = edx.analytics.tasks.warehouse.load_internal_reporting_course_catalog:PullDiscoveryCoursesAPIData

    load-d-certificates = edx.analytics.tasks.warehouse.load_internal_reporting_certificates:LoadInternalReportingCertificatesToWarehouse

    load-d-country = edx.analytics.tasks.warehouse.load_internal_reporting_country:LoadInternalReportingCountryToWarehouse

    load-d-user = edx.analytics.tasks.warehouse.load_internal_reporting_user:LoadInternalReportingUserToWarehouse

    load-d-user-course = edx.analytics.tasks.warehouse.load_internal_reporting_user_course:LoadUserCourseSummary

    load-events = edx.analytics.tasks.warehouse.load_internal_reporting_events:TrackingEventRecordDataTask

    load-f-user-activity = edx.analytics.tasks.warehouse.load_internal_reporting_user_activity:LoadInternalReportingUserActivityToWarehouse

    load-internal-database = edx.analytics.tasks.warehouse.load_internal_reporting_database:ImportMysqlToVerticaTask

    load-internal-active-users = edx.analytics.tasks.warehouse.load_internal_reporting_active_users:LoadInternalReportingActiveUsersToWarehouse

    load-warehouse = edx.analytics.tasks.warehouse.load_warehouse:LoadWarehouseWorkflow

    load-warehouse-bigquery=edx.analytics.tasks.warehouse.load_warehouse_bigquery:LoadWarehouseBigQueryTask

    push_to_vertica_lms_courseware_link_clicked = edx.analytics.tasks.warehouse.lms_courseware_link_clicked:PushToVerticaLMSCoursewareLinkClickedTask

    run-vertica-sql-script = edx.analytics.tasks.warehouse.run_vertica_sql_script:RunVerticaSqlScriptTask

    run-vertica-sql-scripts = edx.analytics.tasks.warehouse.run_vertica_sql_scripts:RunVerticaSqlScriptTask

    test-vertica-sqoop = edx.analytics.tasks.common.vertica_export:VerticaSchemaToBigQueryTask


    # financial:

    cybersource = edx.analytics.tasks.warehouse.financial.cybersource:DailyPullFromCybersourceTask

    ed_services_report = edx.analytics.tasks.warehouse.financial.ed_services_financial_report:BuildEdServicesReportTask

    financial_reports  = edx.analytics.tasks.warehouse.financial.finance_reports:BuildFinancialReportsTask

    orders = edx.analytics.tasks.warehouse.financial.orders_import:OrderTableTask

    payment_reconcile = edx.analytics.tasks.warehouse.financial.reconcile:ReconcileOrdersAndTransactionsTask

    paypal = edx.analytics.tasks.warehouse.financial.paypal:PaypalTransactionsByDayTask


    # export:

    data_obfuscation   = edx.analytics.tasks.export.data_obfuscation:ObfuscatedCourseDumpTask

    dump-student-module = edx.analytics.tasks.export.database_exports:StudentModulePerCourseTask

    events_obfuscation = edx.analytics.tasks.export.events_obfuscation:ObfuscateCourseEventsTask

    export-events = edx.analytics.tasks.export.event_exports:EventExportTask

    export-events-by-course = edx.analytics.tasks.export.event_exports_by_course:EventExportByCourseTask

    export-student-module = edx.analytics.tasks.export.database_exports:StudentModulePerCourseAfterImportWorkflow

    obfuscation = edx.analytics.tasks.export.obfuscation:ObfuscatedCourseTask


    # monitor:

    all_events_report = edx.analytics.tasks.monitor.total_events_report:TotalEventsReportWorkflow

    enrollment_validation = edx.analytics.tasks.monitor.enrollment_validation:CourseEnrollmentValidationTask

    overall_events = edx.analytics.tasks.monitor.overall_events:TotalEventsDailyTask

    noop = edx.analytics.tasks.monitor.performance:ParseEventLogPerformanceTask


    # enterprise:

    enterprise_enrollments = edx.analytics.tasks.enterprise.enterprise_enrollments:ImportEnterpriseEnrollmentsIntoMysql


mapreduce.engine =

    hadoop = edx.analytics.tasks.common.mapreduce:MapReduceJobRunner

    local = luigi.contrib.hadoop:LocalJobRunner

    emu = edx.analytics.tasks.common.mapreduce:EmulatedMapReduceJobRunner


Staff Graded Assignment XBlock

 Дописан функционал создания zip файлов присланных студентами файлов в директории
course.name - course.run - block_id

Пример:
/edx/var/edxapp/media/fa/finadvice/edx_sga_zip/2022/9a0ceeb19/Ivanov.zip

Создание zip запускается так:

./manage.py lms --setting=aws sga_migrate_submissions
--course 'course-v1:fa+finadvice+2022_9_1'
--block 'block-v1:fa+finadvice+2022_9_1+type@edx_sga+block@0a1af775926a41b798b841209af15c75'


Код: кинуть в папку management 

sga_migrate_submissions.py



import json

from django.core.management.base import BaseCommand, CommandError # lint-amnesty, pylint: disable=import-error
from courseware.courses import get_course_by_id # lint-amnesty, pylint: disable=import-error
from courseware.models import StudentModule # lint-amnesty, pylint: disable=import-error
from opaque_keys.edx.keys import CourseKey # lint-amnesty, pylint: disable=import-error
from student.models import anonymous_id_for_user # lint-amnesty, pylint: disable=import-error
from submissions import api as submissions_api # lint-amnesty, pylint: disable=import-error
from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=import-error
from io import BytesIO
import zipfile
import os
import shutil
from opaque_keys.edx.locator import BlockUsageLocator
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
from edx_sga.constants import BLOCK_SIZE, ITEM_TYPE
from edx_sga.tasks import (
get_zip_file_name,
get_zip_file_path,
get_zip_file_dir,
zip_student_submissions,
_get_student_submissions
)

class Command(BaseCommand):
"""
Migrates existing SGA submissions for a course from old SGA implementation
to newer version that uses the 'submissions' application.
sga_migrate_submissions --course 'course-v1:fa+finadvice+2022_1' --block 'block-v1:fa+finadvice+2022_1+type@edx_sga+block@0a1af775926a41b798b841209af15c75'
"""
# args = ["<course_id>","<block_id>"]
def add_arguments(self, parser):
parser.add_argument(
'-c',
'--course',
metavar='COURSE_ID',
dest='course',
default=False,
help='zip course'
)
parser.add_argument(
'-b',
'--block',
metavar='BLOCK_ID',
dest='block',
default=False,
help='zip block'
)

def handle(self, *args, **options):
"""
Migrates existing SGA submissions.
"""
course_id = options['course']
block_id = options['block']
# course_id = 'course-v1:fa+finadvice+2022_9_1'
course_key = CourseKey.from_string(course_id)
course = get_course_by_id(course_key)
# block_id = 'block-v1:fa+finadvice+2022_9_1+type@edx_sga+block@f7324e8b3f3f464e9d532359a0ceeb19'

locator_unicode = block_id
locator = BlockUsageLocator.from_string(locator_unicode)
directory = get_zip_file_dir(locator)
directory_abs = os.path.join('/edx/var/edxapp/media', directory)
direcory_with_run=os.path.join(directory_abs, locator.run)
directory_with_block_id=os.path.join(direcory_with_run, locator.block_id)
if not os.path.exists(direcory_with_run):
os.makedirs(direcory_with_run)

if not os.path.exists(directory_with_block_id):
os.makedirs(directory_with_block_id)

student_submissions = _get_student_submissions(block_id, course_id, locator)
zip_file_bytes = BytesIO()
with zipfile.ZipFile(zip_file_bytes, 'w') as zip_pointer:
for student_username, submission_file_path in student_submissions:
with default_storage.open(submission_file_path, 'rb') as destination_file:
filename_in_zip = '{}_{}'.format(student_username,os.path.basename(submission_file_path))
print filename_in_zip
zip_pointer.writestr(filename_in_zip, destination_file.read())
zip_file_bytes.seek(0)
filename_with_block_id=os.path.join(directory_with_block_id, filename_in_zip +'.zip' )
with open(filename_with_block_id, "wb") as zip_file_pointer:
zip_file_pointer.write(zip_file_bytes.getvalue())
print 'created '+filename_with_block_id


вторник, 29 ноября 2022 г.

Hadoop + hive + Spark

Добавляем файл для дальнейшего анализа  

 /edx/app/hadoop/hadoop/bin/hdfs dfs -put -f /edx/var/log/tracking/tracking.log hdfs://localhost:9000/data/tracking.log

Проверяем

hdfs dfs -ls /data

Запускаем расчёт через подключение к удаленному хосту по ключу

remote-task --host server3 --user vladimir --private-key /home/vladimir/yandex-fa.pem --remote-name analyticstack --skip-setup --wait \ 

AnswerDistributionWorkflow --local-scheduler \

--src hdfs://localhost:9000/data \

--dest  hdfs://localhost:9000/edx-analytics-pipeline \

--name hadoop  --output-root hdfs://localhost:9000/output/ \

--include 'tracking.log.gz'  \

--manifest hdfs://localhost:9000/data/manifest.txt  \

--base-input-format "org.edx.hadoop.input.ManifestTextInputFormat" \

 --lib-jar "hdfs://localhost:9000/edx-analytics-pipeline/packages/edx-analytics-hadoop-util.jar" --n-reduce-tasks 1 \

--marker hdfs://localhost:9000/edx-analytics-pipeline/marker/  \

--credentials "/edx/etc/edx-analytics-pipeline/output.json" 

 

пятница, 18 ноября 2022 г.

Верификация по фото

Как продлить действие верификации до 2025

SQL: 

Update edxapp.verify_student_softwaresecurephotoverification SET submitted_at='2024-12-12 12:12:12.000000', updated_at='2022-11-18 12:12:12.808484' WHERE edxapp.verify_student_softwaresecurephotoverification.submitted_at is not null;


Проверить настройки







 

воскресенье, 30 октября 2022 г.

Почта

 Настройки почты 

    "EMAIL_HOST_PASSWORD": "**********,

    "EMAIL_HOST_USER": "vvandropov@fa.ru",

"DEFAULT_FROM_EMAIL": "vvandropov@fa.ru",

    "EMAIL_BACKEND": "django_smtp_ssl.SSLEmailBackend",

    "EMAIL_HOST": "smtp.mail.ru",

    "EMAIL_PORT": 465,

    "EMAIL_USE_TLS": false,

    "EMAIL_USE_SSL": true,


Настройки  postfix



mydestination = edx3.ru-central1.internal, localhost
relayhost =[smtp.mail.ru]:465
alias_database = hash:/etc/aliases
alias_maps = hash:/etc/aliases
command_directory = /usr/sbin
data_directory = /var/lib/postfix
debug_peer_level = 2
debugger_command = 'PATH=/bin:/usr/bin:/usr/local/bin:/usr/X11R6/bin ddd $daemon_directory/$process_name $process_id & sleep 5'
html_directory = no
inet_interfaces = localhost
inet_protocols = ipv4
mail_owner = postfix
manpage_directory = /usr/share/man
mydomain = edx3.ru-central1.internal
myhostname = edx3.ru-central1.internal
mynetworks = 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16
queue_directory = /var/spool/postfix
setgid_group = postdrop
smtp_tls_security_level = encrypt
smtp_use_tls = yes
unknown_local_recipient_reject_code = 550
sendmail_path = /usr/sbin/sendmail
smtp_tls_CAfile = /etc/ssl/certs/ca-certificates.crt
mailq_path = /usr/bin/mailq
newaliases_path = /usr/bin/newaliases
smtp_generic_maps = hash:/etc/postfix/generic
smtp_sasl_auth_enable = yes
smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd
smtp_sasl_security_options = noanonymous
smtp_tls_wrappermode = yes
compatibility_level = 2


aliase 

root;  vvandropov@fa.ru

generic 

root  mycoolcamera@mail.r

sasl-password 

[smtp.mail.ru]:465 mycoolcamera@mail.ru:*********


commands after

postmap /etc/postfix/sasl_passwd

root@server3:/home/vladimir# postmap /etc/postfix/generic

root@server3:/home/vladimir# systemctl restart postfix 

root@server3:/home/vladimir# postfix reload

root@server3:/home/vladimir# systemctl restart postfix 

root@server3:/home/vladimir# echo "hllo world" |mail -s "tm"  mycoolcamera@mail.ru

root@server3:/home/vladimir# tail -111 /var/log/mail.log