Finam.ml
Financial Machine Learning
вторник, 7 марта 2023 г.
пятница, 13 января 2023 г.
Screenshots Insights.online.fa.ru
пятница, 23 декабря 2022 г.
task: LastCountryOfUser
Географическая метка поьзователей
remote-task --host insights --user vladimir --private-key /home/vladimir/yandex-fa.pem --remote-name analyticstack --skip-setup --wait --local-scheduler \
LastCountryOfUser --interval 2022-01-01-2022-12-12 \
--n-reduce-tasks 1 \
--overwrite-n-days 365
четверг, 22 декабря 2022 г.
task: ModuleEngagementWorkflowTask
remote-task --host insights --user vladimir --private-key /home/vladimir/yandex-fa.pem --local-scheduler --remote-name analyticstack --skip-setup --wait ModuleEngagementWorkflowTask --date $(date +%Y-%m-%d -d "2021-12-12") --indexing-tasks 5 --throttle 0.5 --n-reduce-tasks 1
--local-scheduler - указатель использовать локальный луиджи
hadoop task: ImportEnrollmentsIntoMysql
remote-task --host insights --user vladimir --private-key /home/vladimir/yandex-fa.pem --remote-name analyticstack --skip-setup --wait ImportEnrollmentsIntoMysql --local-scheduler \
--interval 2018-01-01-2018-12-12 \
--n-reduce-tasks 1 \
--overwrite-mysql \
--overwrite-hive --overwrite-n-days 365
host - имя машины которую анализируем
private-key - ключ доступа по ssh
четверг, 1 декабря 2022 г.
analytics.tasks
analytics.tasks
# common
sqoop-import = edx.analytics.tasks.common.sqoop:SqoopImportFromMysql
insert-into-table = edx.analytics.tasks.common.mysql_load:MysqlInsertTask
bigquery-load = edx.analytics.tasks.common.bigquery_load:BigQueryLoadTask
# insights
answer-dist = edx.analytics.tasks.insights.answer_dist:AnswerDistributionPerCourse
calendar = edx.analytics.tasks.insights.calendar_task:CalendarTableTask
course_blocks = edx.analytics.tasks.insights.course_blocks:CourseBlocksApiDataTask
course_list = edx.analytics.tasks.insights.course_list:CourseListApiDataTask
database-import = edx.analytics.tasks.insights.database_imports:ImportAllDatabaseTablesTask
engagement = edx.analytics.tasks.insights.module_engagement:ModuleEngagementDataTask
enrollments = edx.analytics.tasks.insights.enrollments:ImportEnrollmentsIntoMysql
location-per-course = edx.analytics.tasks.insights.location_per_course:LastCountryOfUser
problem_response = edx.analytics.tasks.insights.problem_response:LatestProblemResponseDataTask
tags-dist = edx.analytics.tasks.insights.tags_dist:TagsDistributionPerCourse
user-activity = edx.analytics.tasks.insights.user_activity:InsertToMysqlCourseActivityTask
video = edx.analytics.tasks.insights.video:InsertToMysqlAllVideoTask
# data_api
grade-dist = edx.analytics.tasks.data_api.studentmodule_dist:GradeDistFromSqoopToMySQLWorkflow
student_engagement = edx.analytics.tasks.data_api.student_engagement:StudentEngagementTask
# warehouse:
event-type-dist = edx.analytics.tasks.warehouse.event_type_dist:PushToVerticaEventTypeDistributionTask
load-course-catalog = edx.analytics.tasks.warehouse.load_internal_reporting_course_catalog:PullDiscoveryCoursesAPIData
load-d-certificates = edx.analytics.tasks.warehouse.load_internal_reporting_certificates:LoadInternalReportingCertificatesToWarehouse
load-d-country = edx.analytics.tasks.warehouse.load_internal_reporting_country:LoadInternalReportingCountryToWarehouse
load-d-user = edx.analytics.tasks.warehouse.load_internal_reporting_user:LoadInternalReportingUserToWarehouse
load-d-user-course = edx.analytics.tasks.warehouse.load_internal_reporting_user_course:LoadUserCourseSummary
load-events = edx.analytics.tasks.warehouse.load_internal_reporting_events:TrackingEventRecordDataTask
load-f-user-activity = edx.analytics.tasks.warehouse.load_internal_reporting_user_activity:LoadInternalReportingUserActivityToWarehouse
load-internal-database = edx.analytics.tasks.warehouse.load_internal_reporting_database:ImportMysqlToVerticaTask
load-internal-active-users = edx.analytics.tasks.warehouse.load_internal_reporting_active_users:LoadInternalReportingActiveUsersToWarehouse
load-warehouse = edx.analytics.tasks.warehouse.load_warehouse:LoadWarehouseWorkflow
load-warehouse-bigquery=edx.analytics.tasks.warehouse.load_warehouse_bigquery:LoadWarehouseBigQueryTask
push_to_vertica_lms_courseware_link_clicked = edx.analytics.tasks.warehouse.lms_courseware_link_clicked:PushToVerticaLMSCoursewareLinkClickedTask
run-vertica-sql-script = edx.analytics.tasks.warehouse.run_vertica_sql_script:RunVerticaSqlScriptTask
run-vertica-sql-scripts = edx.analytics.tasks.warehouse.run_vertica_sql_scripts:RunVerticaSqlScriptTask
test-vertica-sqoop = edx.analytics.tasks.common.vertica_export:VerticaSchemaToBigQueryTask
# financial:
cybersource = edx.analytics.tasks.warehouse.financial.cybersource:DailyPullFromCybersourceTask
ed_services_report = edx.analytics.tasks.warehouse.financial.ed_services_financial_report:BuildEdServicesReportTask
financial_reports = edx.analytics.tasks.warehouse.financial.finance_reports:BuildFinancialReportsTask
orders = edx.analytics.tasks.warehouse.financial.orders_import:OrderTableTask
payment_reconcile = edx.analytics.tasks.warehouse.financial.reconcile:ReconcileOrdersAndTransactionsTask
paypal = edx.analytics.tasks.warehouse.financial.paypal:PaypalTransactionsByDayTask
# export:
data_obfuscation = edx.analytics.tasks.export.data_obfuscation:ObfuscatedCourseDumpTask
dump-student-module = edx.analytics.tasks.export.database_exports:StudentModulePerCourseTask
events_obfuscation = edx.analytics.tasks.export.events_obfuscation:ObfuscateCourseEventsTask
export-events = edx.analytics.tasks.export.event_exports:EventExportTask
export-events-by-course = edx.analytics.tasks.export.event_exports_by_course:EventExportByCourseTask
export-student-module = edx.analytics.tasks.export.database_exports:StudentModulePerCourseAfterImportWorkflow
obfuscation = edx.analytics.tasks.export.obfuscation:ObfuscatedCourseTask
# monitor:
all_events_report = edx.analytics.tasks.monitor.total_events_report:TotalEventsReportWorkflow
enrollment_validation = edx.analytics.tasks.monitor.enrollment_validation:CourseEnrollmentValidationTask
overall_events = edx.analytics.tasks.monitor.overall_events:TotalEventsDailyTask
noop = edx.analytics.tasks.monitor.performance:ParseEventLogPerformanceTask
# enterprise:
enterprise_enrollments = edx.analytics.tasks.enterprise.enterprise_enrollments:ImportEnterpriseEnrollmentsIntoMysql
mapreduce.engine =
hadoop = edx.analytics.tasks.common.mapreduce:MapReduceJobRunner
local = luigi.contrib.hadoop:LocalJobRunner
emu = edx.analytics.tasks.common.mapreduce:EmulatedMapReduceJobRunner
Staff Graded Assignment XBlock
Дописан функционал создания zip файлов присланных студентами файлов в директории
course.name - course.run - block_id
Пример:
/edx/var/edxapp/media/fa/finadvice/edx_sga_zip/2022/9a0ceeb19/Ivanov.zip
Создание zip запускается так:
Код: кинуть в папку management
sga_migrate_submissions.py
вторник, 29 ноября 2022 г.
Hadoop + hive + Spark
Добавляем файл для дальнейшего анализа
/edx/app/hadoop/hadoop/bin/hdfs dfs -put -f /edx/var/log/tracking/tracking.log hdfs://localhost:9000/data/tracking.log
Проверяем
hdfs dfs -ls /data
Запускаем расчёт через подключение к удаленному хосту по ключу
remote-task --host server3 --user vladimir --private-key /home/vladimir/yandex-fa.pem --remote-name analyticstack --skip-setup --wait \
AnswerDistributionWorkflow --local-scheduler \
--src hdfs://localhost:9000/data \
--dest hdfs://localhost:9000/edx-analytics-pipeline \
--name hadoop --output-root hdfs://localhost:9000/output/ \
--include 'tracking.log.gz' \
--manifest hdfs://localhost:9000/data/manifest.txt \
--base-input-format "org.edx.hadoop.input.ManifestTextInputFormat" \
--lib-jar "hdfs://localhost:9000/edx-analytics-pipeline/packages/edx-analytics-hadoop-util.jar" --n-reduce-tasks 1 \
--marker hdfs://localhost:9000/edx-analytics-pipeline/marker/ \
--credentials "/edx/etc/edx-analytics-pipeline/output.json"
пятница, 18 ноября 2022 г.
Верификация по фото
Как продлить действие верификации до 2025
SQL:
Update edxapp.verify_student_softwaresecurephotoverification SET submitted_at='2024-12-12 12:12:12.000000', updated_at='2022-11-18 12:12:12.808484' WHERE edxapp.verify_student_softwaresecurephotoverification.submitted_at is not null;
Проверить настройки
воскресенье, 30 октября 2022 г.
Почта
Настройки почты
"EMAIL_HOST_PASSWORD": "**********,
"EMAIL_HOST_USER": "vvandropov@fa.ru",
"DEFAULT_FROM_EMAIL": "vvandropov@fa.ru",
"EMAIL_BACKEND": "django_smtp_ssl.SSLEmailBackend",
"EMAIL_HOST": "smtp.mail.ru",
"EMAIL_PORT": 465,
"EMAIL_USE_TLS": false,
"EMAIL_USE_SSL": true,
Настройки postfix
mydestination = edx3.ru-central1.internal, localhostrelayhost =[smtp.mail.ru]:465alias_database = hash:/etc/aliasesalias_maps = hash:/etc/aliasescommand_directory = /usr/sbindata_directory = /var/lib/postfixdebug_peer_level = 2debugger_command = 'PATH=/bin:/usr/bin:/usr/local/bin:/usr/X11R6/bin ddd $daemon_directory/$process_name $process_id & sleep 5'html_directory = noinet_interfaces = localhostinet_protocols = ipv4mail_owner = postfixmanpage_directory = /usr/share/manmydomain = edx3.ru-central1.internalmyhostname = edx3.ru-central1.internalmynetworks = 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16queue_directory = /var/spool/postfixsetgid_group = postdropsmtp_tls_security_level = encryptsmtp_use_tls = yesunknown_local_recipient_reject_code = 550sendmail_path = /usr/sbin/sendmailsmtp_tls_CAfile = /etc/ssl/certs/ca-certificates.crtmailq_path = /usr/bin/mailqnewaliases_path = /usr/bin/newaliasessmtp_generic_maps = hash:/etc/postfix/genericsmtp_sasl_auth_enable = yessmtp_sasl_password_maps = hash:/etc/postfix/sasl_passwdsmtp_sasl_security_options = noanonymoussmtp_tls_wrappermode = yescompatibility_level = 2
aliase
root; vvandropov@fa.ru
generic
root mycoolcamera@mail.r
sasl-password
[smtp.mail.ru]:465 mycoolcamera@mail.ru:*********
commands after
postmap /etc/postfix/sasl_passwd
root@server3:/home/vladimir# postmap /etc/postfix/generic
root@server3:/home/vladimir# systemctl restart postfix
root@server3:/home/vladimir# postfix reload
root@server3:/home/vladimir# systemctl restart postfix
root@server3:/home/vladimir# echo "hllo world" |mail -s "tm" mycoolcamera@mail.ru
root@server3:/home/vladimir# tail -111 /var/log/mail.log