Commit ec8beb89 authored by Marco Schmiedel's avatar Marco Schmiedel

fix

parent fc69800a
{
"fileId": "1d59cc86-7b89-484d-a6da-2e1563612c68",
"originalPath": "work/routes/EeccxRouter.py",
"currentPath": "work/routes/EeccxRouter.py",
"hash": "92720db32fef845c68e8a7df6e1295371fbe6b757313eca5ab03b7a28d36ad28",
"docContent": "<p><br></p>",
"checkedStatus": "done",
"comments": [],
"lastCheckedTimestamp": 1747070393555,
"lastFileModificationTimestamp": 1747070388417.1619
}
......@@ -2,9 +2,9 @@
"fileId": "22983490-9c01-4bd1-8649-dfe87c659225",
"originalPath": "work/config/MauiConfig.py",
"currentPath": "work/config/MauiConfig.py",
"hash": "6e627f3800fd413c6dbde92ad2e274d5e3047af0f906de4d75fc826cc129631e",
"hash": "08c57a67f7a74d7b702b572da3cd912bf4603ee97e9495a6be2ce60b73beab20",
"docContent": "<p><br></p>",
"checkedStatus": "todo",
"checkedStatus": "done",
"comments": [
{
"commentId": "3bc16f5e-4032-44a8-9012-4b632849ba50",
......@@ -12,6 +12,6 @@
"timestamp": 1744614418809
}
],
"lastCheckedTimestamp": 1746694114141,
"lastFileModificationTimestamp": 1745313945182.1555
"lastCheckedTimestamp": 1747070436322,
"lastFileModificationTimestamp": 1747043388546.054
}
......@@ -2,7 +2,7 @@
"fileId": "36e791b4-e235-42f6-ac61-8560f1762892",
"originalPath": "work/workbench/Workbench.mwb",
"currentPath": "work/workbench/Workbench.mwb",
"hash": "d53db9e9d211116d4aafc32106a7e0c05a86c062af72f21a37420853a1c4eacc",
"hash": "afea3df7165b4dad78d5a9d92ede4fec601f68ed0a14147532cf1ca00617c29e",
"docContent": "<p><br></p>",
"checkedStatus": "done",
"comments": [
......@@ -12,6 +12,6 @@
"timestamp": 1746693753181
}
],
"lastCheckedTimestamp": 1746693747974,
"lastFileModificationTimestamp": 1746440499172.53
"lastCheckedTimestamp": 1747070021483,
"lastFileModificationTimestamp": 1747068819870.2563
}
......@@ -2,9 +2,9 @@
"fileId": "38b9eebe-955e-4052-a0f6-29c69b1242b3",
"originalPath": "work/config/MysqlConfig.py",
"currentPath": "work/config/MysqlConfig.py",
"hash": "8eeae892f7c5f5aa1e894ca9ff7b8c66ea2891bc37c0167c404cd6e0cb95f858",
"hash": "d8958dba0bf7c100587dabf6ff576e0a1905a0aed4980a6c64ff6254f3671e5a",
"docContent": "<p><br></p>",
"checkedStatus": "todo",
"checkedStatus": "done",
"comments": [
{
"commentId": "56c5adba-20f4-4524-a894-41f81ab7ca55",
......@@ -12,6 +12,6 @@
"timestamp": 1744622354948
}
],
"lastCheckedTimestamp": 1745314583521,
"lastFileModificationTimestamp": 1745313973064.8933
"lastCheckedTimestamp": 1747070439965,
"lastFileModificationTimestamp": 1747070428651.6106
}
{
"fileId": "48126029-3c3e-4372-9f3e-1e8b9686114e",
"originalPath": "work/commands/importCacheToDatabase.py",
"currentPath": "work/commands/importCacheToDatabase.py",
"hash": "3c19dde87c665d72591ff9391a0dd6ec28218002116df06b22217a09d2a73e27",
"docContent": "<p><br></p>",
"checkedStatus": "done",
"comments": [],
"lastCheckedTimestamp": 1747070815750,
"lastFileModificationTimestamp": 1747070802673.0312
}
......@@ -2,9 +2,9 @@
"fileId": "4c784f14-4710-4694-bf73-f5665baab43f",
"originalPath": "work/cron.sh",
"currentPath": "work/cron.sh",
"hash": "4c6e694c417005a79207a32c26609e0e2701f17d6484a536bd8188bf8dcceb93",
"hash": "8950b2d4895462785979d16bc1db8830ed40cca30d60a680df360f731b95baa9",
"docContent": "<p><br></p>",
"checkedStatus": "todo",
"checkedStatus": "done",
"comments": [
{
"commentId": "e5f40597-ae51-440f-886a-44f06dbe8e96",
......@@ -12,6 +12,6 @@
"timestamp": 1746693690181
}
],
"lastCheckedTimestamp": 1746693667833,
"lastFileModificationTimestamp": 1746448049902.1914
"lastCheckedTimestamp": 1747070016564,
"lastFileModificationTimestamp": 1747070003854.197
}
......@@ -2,10 +2,10 @@
"fileId": "58307c8c-416a-4c24-adc9-7ed6324d1f8a",
"originalPath": "work/manager/WebManager.py",
"currentPath": "work/manager/WebManager.py",
"hash": "66f022cdc155ded9c47e49a893ed3070099faa960a72aa01d23929a1c02a8657",
"hash": "5a987f9d37c2d083c9a04ea0a0c4739a4fbd6c2e6c98877a0f64fadb45717a1c",
"docContent": "<p><br></p>",
"checkedStatus": "changed",
"checkedStatus": "done",
"comments": [],
"lastCheckedTimestamp": 1746694408088,
"lastFileModificationTimestamp": 1746696251620.3523
"lastCheckedTimestamp": 1747070585799,
"lastFileModificationTimestamp": 1747070580506.2974
}
{
"fileId": "5f874bee-40e2-4b9a-b102-f0b6d643a840",
"originalPath": "work/commands/downloadDataFromMaui.py",
"currentPath": "work/commands/downloadDataFromMaui.py",
"hash": "f75e0013a123055434a4592bb3509d3ff298273727226ef431662f3fad739fad",
"docContent": "<p><br></p>",
"checkedStatus": "done",
"comments": [],
"lastCheckedTimestamp": 1747071242419,
"lastFileModificationTimestamp": 1747071185164.952
}
{
"fileId": "62aea232-2549-437e-b5a9-72cb2aa92d16",
"originalPath": "work/commands/calculateTarifDetailsWithGpt.py",
"currentPath": "work/commands/calculateTarifDetailsWithGpt.py",
"hash": "9161246779e6b04e4ae512afe91cfae14ad7e9fc28395d42f54627e3b70a25b0",
"docContent": "<p><br></p>",
"checkedStatus": "done",
"comments": [],
"lastCheckedTimestamp": 1747071244862,
"lastFileModificationTimestamp": 1747071237273.2832
}
......@@ -2,9 +2,9 @@
"fileId": "647ff9a8-a56f-486e-ba2a-8ff77e4514d4",
"originalPath": "work/Dockerfile",
"currentPath": "work/Dockerfile",
"hash": "d885a8a45174b2f425d3c0201b797754c69f6ff798dabd51f0e53af17b047964",
"hash": "ca6a37e37aff3fff276f8020d9860b94c6556181bbb65a3fe1c62f3868f7f0b3",
"docContent": "<p><br></p>",
"checkedStatus": "changed",
"checkedStatus": "done",
"comments": [
{
"commentId": "2a07c637-2149-4d5a-870d-94870f78945d",
......@@ -12,6 +12,6 @@
"timestamp": 1746693591017
}
],
"lastCheckedTimestamp": 1746693552978,
"lastFileModificationTimestamp": 1746694865448.947
"lastCheckedTimestamp": 1747069787674,
"lastFileModificationTimestamp": 1747069781547.9219
}
......@@ -2,9 +2,9 @@
"fileId": "766dc461-001e-4901-8faf-263820ad96cd",
"originalPath": "work/manager/MysqlManager.py",
"currentPath": "work/manager/MysqlManager.py",
"hash": "27129c35df4b6b0e4d5fcb7a77c8e1c19d1b74f80d5c3ec822cdc26701124a68",
"hash": "9a9ca8572ad133ef4a191b7082ffb025d979f84dce2109ff5be33108cb807652",
"docContent": "<p><br></p>",
"checkedStatus": "changed",
"checkedStatus": "done",
"comments": [
{
"commentId": "7227a7a0-99bc-47b4-a725-3547eb56015d",
......@@ -12,7 +12,7 @@
"timestamp": 1746694262639
}
],
"lastCheckedTimestamp": 1745314589383,
"lastFileModificationTimestamp": 1746696474493.3755,
"lastCheckedTimestamp": 1747070649128,
"lastFileModificationTimestamp": 1747070643843.446,
"flaggedForCopy": false
}
{
"fileId": "78db1316-a768-4c1f-b15c-7a408444a030",
"originalPath": "work/routes/HealtCheckRouter.py",
"currentPath": "work/routes/HealtCheckRouter.py",
"hash": "965774cdb8edb7b68ec0341e4d122765853c5fa82b5432df2843234c25874f3d",
"docContent": "<p><br></p>",
"checkedStatus": "done",
"comments": [],
"lastCheckedTimestamp": 1747070215651,
"lastFileModificationTimestamp": 1747070210301.3403
}
......@@ -2,9 +2,9 @@
"fileId": "7a3a246b-fc0e-4c80-b748-96b941efab5c",
"originalPath": "work/config/AWSConfig.py",
"currentPath": "work/config/AWSConfig.py",
"hash": "5a6654cb1cd77f8d531fcc1541d31261ea02c4e8cb126f2cc43a217c9c6920aa",
"hash": "29bc59fd6ecbf98aa7efcfa3ef371bb912ac144f077895c279bb80ba150ee734",
"docContent": "<p><br></p>",
"checkedStatus": "todo",
"checkedStatus": "done",
"comments": [
{
"commentId": "3c070677-67c2-458d-8ad9-1ef595c16e0e",
......@@ -12,6 +12,6 @@
"timestamp": 1746694106055
}
],
"lastCheckedTimestamp": 1745314580866,
"lastFileModificationTimestamp": 1745311719614.9841
"lastCheckedTimestamp": 1747070433793,
"lastFileModificationTimestamp": 1747042244230.3608
}
{
"fileId": "8c1b7b54-86c0-453c-839c-95390d883819",
"originalPath": "work/commands/uploadCacheToAwsS3.py",
"currentPath": "work/commands/uploadCacheToAwsS3.py",
"hash": "158bb6839fc011bfeb8ab54d335f897c629228f2169844f2f6118f803b80c64f",
"docContent": "<p><br></p>",
"checkedStatus": "done",
"comments": [],
"lastCheckedTimestamp": 1747070735035,
"lastFileModificationTimestamp": 1747070723183.9927
}
......@@ -2,16 +2,21 @@
"fileId": "986eeb57-8634-4f40-a4ea-a2eae9d87e71",
"originalPath": "work/readme.md",
"currentPath": "work/readme.md",
"hash": "3e2bf4db6ad284fb011128f2ac0d3cf7849268068a39b160418173f0230ba4bd",
"hash": "455ce9ea71460c0f1f2b43ad6ea5c9706a4f2003e46d60622086b7cab47925db",
"docContent": "<p><br></p>",
"checkedStatus": "changed",
"checkedStatus": "todo",
"comments": [
{
"commentId": "574b8332-b3c0-4afa-9f2b-8a632e910e0d",
"text": "I need to insert the AWS-ECR-Uplink-Data.",
"timestamp": 1746693537936
},
{
"commentId": "e5d599b4-3080-4638-b0a8-753fb4dd3c9b",
"text": "Only the video tutorials are missing...",
"timestamp": 1747069658074
}
],
"lastCheckedTimestamp": 1746693903209,
"lastFileModificationTimestamp": 1746694946510.8994
"lastCheckedTimestamp": 1747069646363,
"lastFileModificationTimestamp": 1747069621488.916
}
{
"fileId": "b71a7bf5-594a-4ac1-9113-25158c35bcb4",
"originalPath": "work/models/token_toke.py",
"currentPath": "work/models/token_toke.py",
"hash": "609e9fec7718b6125d047c7a8c7029b7d0cd2b95df889334914ce058091176bd",
"docContent": "<p><br></p>",
"checkedStatus": "done",
"comments": [],
"lastCheckedTimestamp": 1747070070388,
"lastFileModificationTimestamp": 1747068898597.5894
}
......@@ -2,9 +2,9 @@
"fileId": "caf03c7b-60d8-4a77-ac21-0eccabeae4a2",
"originalPath": "work/boot.sh",
"currentPath": "work/boot.sh",
"hash": "d665dba2f614cbf283cf1900c259bea8472f31353be894740e06535e6c3936c3",
"hash": "9d08025500b916fe294de7aa8b533c29e02743c714022415ae9274e066c4fa6a",
"docContent": "<p><br></p>",
"checkedStatus": "todo",
"checkedStatus": "done",
"comments": [
{
"commentId": "6ba2875c-14b5-4444-a34e-52295efd65bc",
......@@ -12,6 +12,6 @@
"timestamp": 1746693713037
}
],
"lastCheckedTimestamp": 1746693711224,
"lastFileModificationTimestamp": 1746447735575.9163
"lastCheckedTimestamp": 1747070094947,
"lastFileModificationTimestamp": 1747070086824.1516
}
{
"fileId": "e4ffba94-a5e6-40d2-a63e-5bfa60e3d719",
"originalPath": "work/routes/BaseRouter.py",
"currentPath": "work/routes/BaseRouter.py",
"hash": "e8f5cd3137261214985789cc7e328848d9ff379525c706b9acf6ee7a6ea3adec",
"docContent": "<p><br></p>",
"checkedStatus": "done",
"comments": [],
"lastCheckedTimestamp": 1747070513561,
"lastFileModificationTimestamp": 1747070506645.0361
}
......@@ -4,7 +4,7 @@
"currentPath": "work/config/OpenAiConfig.py",
"hash": "50c0f7d96f9ea76aa069a0a24137e898dbd4fc3c4af867565c90468981bf6ff5",
"docContent": "<p><br></p>",
"checkedStatus": "todo",
"checkedStatus": "done",
"comments": [
{
"commentId": "1b2c6a64-0a75-4763-9613-12634d96bed2",
......@@ -12,6 +12,6 @@
"timestamp": 1746694100919
}
],
"lastCheckedTimestamp": 1746694087733,
"lastCheckedTimestamp": 1747070442728,
"lastFileModificationTimestamp": 1746437070245.503
}
# Wir verwenden Ubuntu als Betriebssystem.
# The Docker container uses Ubuntu 24.04 as the base operating system.
FROM ubuntu:24.04
# Wir deaktivieren das interaktive Frontend.
# The non-interactive frontend is configured to suppress installation prompts.
ENV DEBIAN_FRONTEND=noninteractive
# Zuerst aktualisieren wir die Paketquellen und führen ein Upgrade durch.
# The package index files are updated to obtain the latest package metadata.
RUN apt-get -y update
# All installed packages are upgraded to their most recent versions.
RUN apt-get -y upgrade
# Anschließend installieren wir systemweite Hilfspakete.
# The “software-properties-common” package is installed to enable PPA management.
RUN apt-get install -y software-properties-common
# Wir installieren Python3 und pip.
# Python 3 is installed to provide the runtime environment for Python applications.
RUN apt-get install -y python3
# pip is installed to manage Python packages within the container.
RUN apt-get install -y python3-pip
# Wir fügen das PPA hinzu, um die deb-basierte Version von Chromium zu erhalten.
# The xtradeb PPA is added so a deb-based build of Chromium can be installed.
RUN add-apt-repository ppa:xtradeb/apps -y
# Wir aktualisieren erneut die Paketquellen (dadurch werden auch die PPA-Pakete verfügbar).
# The package index files are refreshed to include packages from the newly added PPA.
RUN apt-get -y update
# Wir installieren den Browser.
# The Chromium browser is installed to allow headless or automated browsing.
RUN apt-get install -y chromium-browser
# The Chromium WebDriver is installed to enable Selenium interactions with Chromium.
RUN apt-get install -y chromium-driver
# The Firefox Gecko WebDriver is installed to enable Selenium interactions with Firefox.
RUN apt-get install -y firefox-geckodriver
# Wir entfernen snapd, damit keine Snap-Version von Chromium verwendet wird.
# The “snapd” package is removed to prevent the Snap version of Chromium from being used.
RUN apt-get remove -y snapd
# Wir installieren Cron.
# The cron utility is installed so scheduled tasks can run inside the container.
RUN apt-get install -y cron
# Wir installieren Vim.
# The Vim text editor is installed for in-container file editing.
RUN apt-get install -y vim
# Wir installhieren htop.
# The htop process viewer is installed to facilitate real-time resource monitoring.
RUN apt-get install -y htop
# Wir installhieren ffmpeg.
# The FFmpeg multimedia framework is installed for audio and video processing tasks.
RUN apt-get install -y ffmpeg
# Wir installhieren curl.
# The curl command-line tool is installed to enable data transfer over various protocols.
RUN apt-get install -y curl
# Wir installieren die Python-Abhängigkeiten via pip.
# The Selenium package is installed to drive browser automation from Python.
RUN pip3 install --break-system-packages selenium
# The requests package is installed to simplify HTTP requests in Python.
RUN pip3 install --break-system-packages requests
# SQLAlchemy is installed to provide an ORM for database access.
RUN pip3 install --break-system-packages sqlalchemy
# PyMySQL is installed as a MySQL client library for Python.
RUN pip3 install --break-system-packages pymysql
# pandas is installed to enable data analysis and manipulation.
RUN pip3 install --break-system-packages pandas
# BeautifulSoup 4 is installed to facilitate HTML and XML parsing.
RUN pip3 install --break-system-packages bs4
# feedparser is installed to parse RSS and Atom feeds.
RUN pip3 install --break-system-packages feedparser
# demjson3 is installed to work with JSON that may not strictly follow the standard.
RUN pip3 install --break-system-packages demjson3
# Flask is installed (ignoring any previously installed version) to run the web server.
RUN pip3 install --break-system-packages --ignore-installed flask
# feedgen is installed to generate RSS and Atom feeds programmatically.
RUN pip3 install --break-system-packages feedgen
# boto3 is installed to access AWS services from Python.
RUN pip3 install --break-system-packages boto3
# pydub is installed to handle audio file manipulation.
RUN pip3 install --break-system-packages pydub
# json5 is installed to parse and generate JSON5 data.
RUN pip3 install --break-system-packages json5
# pyotp is installed to generate and verify one-time passwords.
RUN pip3 install --break-system-packages pyotp
# sshtunnel is installed to create SSH tunnels from Python.
RUN pip3 install --break-system-packages sshtunnel
# pypdf is installed to manipulate PDF files from Python.
RUN pip3 install --break-system-packages pypdf
# Wir kopieren die Cron-Datei in den Container.
# The cron configuration file is copied into the appropriate directory inside the container.
COPY config/_CronConfig.txt /etc/cron.d/scrapeNewsCron
# The permissions of the cron configuration file are set to be readable by cron.
RUN chmod 0644 /etc/cron.d/scrapeNewsCron
# The cron configuration is registered so the scheduled tasks become active.
RUN crontab /etc/cron.d/scrapeNewsCron
# The shell script executed by cron is copied into the container.
COPY cron.sh /maui/cron.sh
# The cron shell script is marked as executable.
RUN chmod +x /maui/cron.sh
# Wir kopieren alle Systemdatein in den Container.
# The configuration directory is copied into the container to provide system settings.
COPY config /maui/config
# The manager directory is copied into the container to provide management utilities.
COPY manager /maui/manager
# The commands directory is copied into the container to provide command-line tools.
COPY commands /maui/commands
# The models directory is copied into the container to provide ORM models.
COPY models /maui/models
# The routes directory is copied into the container to provide web route definitions.
COPY routes /maui/routes
# The boot script is copied into the container to serve as the container’s entry point.
COPY boot.sh /maui/boot.sh
# The boot script is marked as executable so it can be run as the container’s default command.
RUN chmod +x /maui/boot.sh
# Wir definieren das Startscript des Containers.
CMD ["/maui/boot.sh"]
\ No newline at end of file
# The boot script is set as the default command that runs when the container starts.
CMD ["/maui/boot.sh"]
#!/bin/bash
set -e
# Dieser Befehl startet den Cron-Service.
# This command starts the cron service.
service cron start
# Dieser Befehl wechselt in das Arbeitsverzeichnis /obsidian/manager.
# This command changes into the application manager directory.
cd /maui/manager
# Dieser Befehl setzt die Umgebungsvariable PYTHONPATH auf /obsidian.
# This command exports the project root so Python modules can be resolved.
export PYTHONPATH=/maui
# Dieser Befehl startet den ApiManager Webserver.
# This command launches the API manager web server.
python3 WebManager.py
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
sys.path.append("..")
# This script scans a local cache directory for PDF files belonging to base tariffs, uploads the PDFs to S3, writes the resulting URLs back into the MySQL database, and logs progress as well as errors to stdout.
import sys; sys.path.append("..")
import os
import datetime
from manager.S3Manager import S3Manager
......@@ -10,100 +7,113 @@ from manager.MysqlManager import MysqlManager
from models.base_base import BaseBase
from models.deal_deal import DealDeal
from models.option_opti import OptionOpti
from models.provisiongroup_pgro import ProvisiongroupPgro # zwingend, um Abhängigkeits-Mapping zu initialisieren
from models.provisiongroup_pgro import ProvisiongroupPgro # zwingend, um Abhängigkeits-Mapping zu initialisieren
#
# Dieses Verzeichnis enthält sämtliche PDF-Dateien für den Upload.
# The variable "cacheDir" stores the file‑system path that contains the PDF files waiting for upload.
cacheDir = "../cache"
#
# Dieses Objekt übernimmt das Hochladen der Dateien in den S3-Bucket und liefert die endgültige URL.
# The variable "s3Manager" holds an instance that encapsulates S3 upload functionality.
s3Manager = S3Manager()
#
# Diese Datenbank-Session ermöglicht Abfragen und Aktualisierungen innerhalb der MySQL-Datenbank.
# The variable "dbSession" stores a SQLAlchemy session used to query and update the MySQL database.
dbSession = MysqlManager().getSession()
#
# Diese Liste sammelt alle PDF-Dateinamen im Cache-Verzeichnis.
# The variable "pdfFiles" gathers all file names inside the cache directory whose names end with the ".pdf" extension, case‑insensitive.
pdfFiles = [f for f in os.listdir(cacheDir) if f.lower().endswith(".pdf")]
#
# Diese Menge speichert alle eindeutigen Basis-IDs, die durch Suffix-Prüfung ermittelt wurden.
# The variable "pdfIdSet" collects distinct base identifiers by stripping the *_flyer or *_pib suffix from each file name stem.
pdfIdSet = set()
for name in pdfFiles:
# The variable "stem" stores the file name without its extension and is converted to lowercase for uniformity.
stem = name[:-4].lower()
# This branch adds the identifier to the set when the file name ends with the flyer suffix.
if stem.endswith("_flyer"):
pdfIdSet.add(stem[:-6])
# This branch adds the identifier to the set when the file name ends with the PIB suffix.
elif stem.endswith("_pib"):
pdfIdSet.add(stem[:-4])
#
# Dieser Block beendet das Skript, wenn keine geeigneten PDF-Dateien vorhanden sind.
# This branch terminates the script early when no matching PDF pairs were found in the cache directory.
if not pdfIdSet:
print(f"INFO: Keine PDF-Paare in '{cacheDir}' gefunden.")
dbSession.close()
sys.exit(0)
#
# Diese Schleife verarbeitet jede erkannte Basis-ID in sortierter Reihenfolge.
# This loop iterates over each distinct base identifier in sorted order to process the associated PDFs.
for currentId in sorted(pdfIdSet):
# The print statement marks the beginning of processing for the current identifier.
print(f"\n--- Verarbeitung ID: {currentId} ---")
#
# Diese Abfrage liefert alle BaseBase-Datensätze, um Mehrfachtreffer sicher zu unterstützen.
# The variable "baseRecords" retrieves all BaseBase rows with a matching provider code so that multiple matches are processed consistently.
baseRecords = dbSession.query(BaseBase).filter_by(providercode_base=currentId).all()
# This branch skips the current identifier when no matching BaseBase row exists.
if not baseRecords:
print(f"WARNUNG: Kein BaseBase-Eintrag für providercode_base='{currentId}'.")
continue
#
# Dieser Pfad verweist auf die potenzielle Flyer-Datei der aktuellen Basis-ID.
# The variable "flyerPath" composes the absolute path to the flyer PDF for the current identifier.
flyerPath = os.path.join(cacheDir, f"{currentId}_flyer.pdf")
#
# Dieser Pfad verweist auf die potenzielle PIB-Datei der aktuellen Basis-ID.
# The variable "pibPath" composes the absolute path to the PIB PDF for the current identifier.
pibPath = os.path.join(cacheDir, f"{currentId}_pib.pdf")
#
# Diese Variable hält die hochgeladene Flyer-URL oder bleibt None, falls kein Upload erfolgte.
# The variable "flyerUrl" will store the public S3 URL of the flyer PDF or remain None when the upload fails or the file does not exist.
flyerUrl = None
# This branch uploads the flyer PDF when the file exists.
if os.path.exists(flyerPath):
# The variable "flyerKey" determines the destination key inside the S3 bucket.
flyerKey = f"flyers/{currentId}_flyer.pdf"
# The variable "flyerUrl" receives the URL returned by the upload method.
flyerUrl = s3Manager.uploadFile(flyerPath, flyerKey)
# This branch prints an error message when the upload failed and no URL was returned.
if not flyerUrl:
print(f"FEHLER: Flyer-Upload fehlgeschlagen für ID {currentId}")
#
# Diese Variable hält die hochgeladene PIB-URL oder bleibt None, falls kein Upload erfolgte.
# The variable "pibUrl" will store the public S3 URL of the PIB PDF or remain None when the upload fails or the file does not exist.
pibUrl = None
# This branch uploads the PIB PDF when the file exists.
if os.path.exists(pibPath):
# The variable "pibKey" determines the destination key inside the S3 bucket.
pibKey = f"pibs/{currentId}_pib.pdf"
# The variable "pibUrl" receives the URL returned by the upload method.
pibUrl = s3Manager.uploadFile(pibPath, pibKey)
# This branch prints an error message when the upload failed and no URL was returned.
if not pibUrl:
print(f"FEHLER: PIB-Upload fehlgeschlagen für ID {currentId}")
#
# Diese Schleife aktualisiert jede gefundene Base-Zeile, um Flyer- und PIB-URLs konsistent zu setzen.
# This loop updates each BaseBase record so that both flyer and PIB URLs are stored without overwriting existing values.
for base in baseRecords:
# This branch writes the flyer URL into the database row when no URL has been stored before and a new URL is available.
if base.flyerurl_base is None and flyerUrl:
base.flyerurl_base = flyerUrl
base.updated_base = datetime.datetime.now()
print(f"INFO: flyerurl_base gesetzt: {flyerUrl}")
# This branch writes the PIB URL into the database row when no URL has been stored before and a new URL is available.
if base.piburl_base is None and pibUrl:
base.piburl_base = pibUrl
base.updated_base = datetime.datetime.now()
print(f"INFO: piburl_base gesetzt: {pibUrl}")
#
# Dieser Aufruf speichert alle Änderungen für die aktuelle Basis-ID atomar in der Datenbank.
# The commit call atomically persists all changes performed for the current identifier.
dbSession.commit()
#
# Hier wird die Session geschlossen, sobald alle Basis-IDs verarbeitet wurden.
# The database session is closed after all identifiers have been processed.
dbSession.close()
#
# Diese Meldung bestätigt das erfolgreiche Ende des gesamten Upload-Vorgangs.
print("INFO: Upload-Vorgang abgeschlossen.")
# The print statement confirms that the entire upload sequence finished successfully.
print("INFO: Upload-Vorgang abgeschlossen.")
\ No newline at end of file
......@@ -8,4 +8,4 @@ USE_SSH_TUNNEL = True
SSH_HOST = "jumphost.bugsmasher.online"
SSH_PORT = 22
SSH_USERNAME = "root"
SSH_PASSWORD = "7dHz2xO8ct1143T"
\ No newline at end of file
SSH_PASSWORD = "7dHz2xO8ct1143T"
#!/bin/bash
# Dieser Wrapper wechselt ins Verzeichnis /maui/commands und startet das
# gewünschte Python-Skript (mit python3), sofern nicht bereits eine Instanz
# dieses Skripts läuft. Gleichzeitig werden alle Ausgaben in zwei getrennten
# Logfiles im Verzeichnis /maui/logs abgelegt, wobei jedes Skript einen
# eigenen Unterordner erhält (benannt nach dem Skriptnamen ohne Erweiterung)
# und die Logfiles die Namen im Format
# - L_yyyymmdd-hhiiss.txt für die Standardausgabe,
# - E_yyyymmdd-hhiiss.err für die Fehlermeldung
# tragen. Logfiles, die älter als 24 Stunden (1440 Minuten) sind, werden
# automatisch gelöscht.
#--- Parameterprüfung ---
# In dieser Abfrage wird überprüft, ob mindestens ein Parameter übergeben wurde.
# This script guarantees that only one instance of a specified Python job runs simultaneously, captures its standard and error output in timestamped log files, and notifies a monitoring endpoint when errors occur.
# The following conditional branch checks whether at least one positional argument has been provided; if not, usage information is printed and the script terminates with exit status 1.
if [ "$#" -lt 1 ]; then
# Hier wird ein Hinweis ausgegeben, wie dieses Skript zu nutzen ist, wenn nicht
# genügend Parameter übergeben wurden.
# The echo command prints usage instructions when no job file is supplied.
echo "Usage: $0 <jobfilename> [arguments...]"
# Dieser Befehl beendet das Skript mit einem Fehlercode.
# The script terminates with exit status 1 when the required argument is missing.
exit 1
fi
# Diese Variable speichert den ersten übergebenen Parameter als Namen des
# Python-Skripts.
# The variable “jobname” stores the first positional argument as the Python job file name.
jobname="$1"
# Dieser Befehl entfernt den ersten Parameter aus der Parameterliste, damit
# weitere Argumente optional weiterverarbeitet werden können.
# The shift statement removes the first positional argument so that any additional arguments remain accessible.
shift
#--- Arbeitsverzeichnis und Log-Verzeichnis festlegen ---
# Diese Variable legt das Arbeitsverzeichnis fest, in dem sich die
# Python-Skripte befinden.
# The variable “WORKDIR” defines the directory where the Python job files reside.
WORKDIR="/maui/commands"
# Diese Variable legt das Hauptverzeichnis für die Logdateien fest.
# The variable “LOG_ROOT” defines the root directory where log folders will be created.
LOG_ROOT="/maui/logs"
# Diese Variable ermittelt aus dem übergebenen Skriptnamen
# (z. B. rawFromBloomberg.py) den Basisteil (rawFromBloomberg).
# The variable “job_base” extracts the base name of the job without its extension.
job_base=$(basename "$jobname" .py)
# Diese Variable bildet den Pfad für das individuellen Logverzeichnis, basierend
# auf dem Basisteil des Skriptnamens.
# The variable “LOG_DIR” composes the path to the job-specific log directory.
LOG_DIR="$LOG_ROOT/$job_base"
# Dieser Befehl stellt sicher, dass das Haupt-Logverzeichnis und das Verzeichnis
# für das aktuelle Skript existieren, und legt sie gegebenenfalls an.
# The mkdir command ensures that the root log directory and the job-specific directory exist, creating them if necessary.
mkdir -p "$LOG_DIR"
# Dieser Befehl findet und löscht alle Logdateien im spezifischen Verzeichnis,
# die älter als 24 Stunden (1440 Minuten) sind.
# The find command removes log files older than twenty-four hours (1 440 minutes) from the job-specific directory.
find "$LOG_DIR" -type f -mmin +1440 -delete
#--- Prozessüberprüfung ---
# Diese Variable speichert die Prozess-ID des aktuell ausgeführten Skripts,
# damit es sich nicht selbst erkennt.
# The variable “current_pid” stores the process identifier of the currently running wrapper instance.
current_pid=$$
# Diese Variable hält den Namen dieses Wrapperskripts (cron.sh), um ihn ebenfalls
# von der Prozessliste auszuschließen.
# The variable “wrapper_name” stores the file name of this wrapper to exclude it from the process search.
wrapper_name=$(basename "$0")
# Diese Variable legt fest, nach welchem exakten Aufrufmuster
# (python3 <jobname>) in der Prozessliste gesucht werden soll.
# The variable “pattern” stores the exact command signature that identifies a running job process.
pattern="python3 $jobname"
# In dieser Variable werden alle zum Muster passenden Prozess-IDs gespeichert,
# wobei Zeilen des Wrappers und greps ausgeschlossen werden.
# The variable “running” captures the process identifiers that match the command signature while excluding grep and wrapper processes.
running=$(ps ax -o pid,cmd | grep "$pattern" | grep -v grep | grep -v "$wrapper_name" | awk '{print $1}')
# Diese Abfrage prüft, ob ein passender Prozess bereits läuft.
# The following conditional branch checks whether at least one matching process identifier was found; if a job is already running, the script informs the user and terminates with exit status 0.
if [ -n "$running" ]; then
# Hier wird der Nutzer informiert, dass der entsprechende Job bereits ausgeführt
# wird, und ein erneuter Start verhindert.
# The echo command informs the user that the requested job is already running.
echo "Job '$jobname' läuft bereits (PID(s): $running). Abbruch."
# Das Skript wird hier mit Exit-Code 0 (ohne Fehler) beendet, um keine neue
# Instanz zu starten.
# The script terminates with exit status 0 to prevent a second instance from starting.
exit 0
fi
#--- Logging vorbereiten und Job starten ---
# Diese Variable erzeugt einen Zeitstempel im Format yyyymmdd-hhiiss
# (z. B. 20250413-114530), um eindeutige Logdateien zu erstellen.
# The variable “timestamp” records the current date and time in YYYYMMDD-HHMMSS format.
timestamp=$(date "+%Y%m%d-%H%M%S")
# Diese Variable bildet den vollständigen Pfad zur Logdatei für die Standardausgabe.
# The variable “STDOUT_LOG” composes the full path to the log file for standard output.
STDOUT_LOG="$LOG_DIR/L_${timestamp}.txt"
# Diese Variable bildet den vollständigen Pfad zur Logdatei für die Fehlermeldungen.
# The variable “ERROR_LOG” composes the full path to the log file for error output.
ERROR_LOG="$LOG_DIR/E_${timestamp}.err"
# Dieser Befehl wechselt in das festgelegte Arbeitsverzeichnis oder bricht mit
# Fehlermeldung ab, falls es nicht erreichbar ist.
# The cd command changes into the working directory or aborts with an error message if the directory is inaccessible.
cd "$WORKDIR" || { echo "Arbeitsverzeichnis $WORKDIR nicht erreichbar." >&2; exit 1; }
# Dieser Befehl führt das Python-Skript aus und leitet stdout in das L_-Logfile
# und stderr in das E_-Logfile um.
# The python3 command executes the specified job, redirecting standard output and error output to dedicated log files.
python3 "$jobname" "$@" > "$STDOUT_LOG" 2> "$ERROR_LOG"
# Fehler senden bei Inhalt: Jobname, Zeilenumbruch, Fehler
# The following conditional branch checks whether the error log file contains data; if errors exist, the job name and error content are posted to the monitoring endpoint.
if [ -s "$ERROR_LOG" ]; then
payload="$jobname
$(<"$ERROR_LOG")"
# The variable “payload” concatenates the job name and error log content for notification.
payload="$jobname : $(<"$ERROR_LOG")"
# The curl command posts the payload to the ntfy.sh endpoint for error reporting.
curl -s -X POST https://ntfy.sh/itmaxDebug -d "$payload"
fi
\ No newline at end of file
fi
import sys
sys.path.append("..")
import sys; sys.path.append("..")
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import QueuePool
import config.MysqlConfig as DatabaseConfig
from sshtunnel import SSHTunnelForwarder
# Diese Klasse verwaltet die MySQL‑Verbindung und nutzt optional einen SSH‑Tunnel.
# Die Konfiguration stammt weiterhin aus config.MysqlConfig; es wird lediglich
# der erweiterte Engine‑Teil (QueuePool, Timeouts u. a.) integriert.
# This class manages the MySQL connection, optionally establishes an SSH tunnel, and exposes a ready-to-use SQLAlchemy session.
class MysqlManager:
# The constructor loads configuration, conditionally creates an SSH tunnel, builds an SQLAlchemy engine with a queue pool, and instantiates the first session.
def __init__(self):
# ───────────────────────────────────────────────────────
# Konfiguration aus dem Modul laden
# ───────────────────────────────────────────────────────
# The variable “dbConfig” stores database connection parameters that are loaded from the configuration module.
self.dbConfig = {
"host": DatabaseConfig.MYSQL_HOST,
"user": DatabaseConfig.MYSQL_USER,
......@@ -25,10 +20,10 @@ class MysqlManager:
"port": DatabaseConfig.MYSQL_PORT,
}
# ───────────────────────────────────────────────────────
# Optionalen SSH‑Tunnel aufbauen
# ───────────────────────────────────────────────────────
# This conditional branch builds an SSH tunnel when the configuration flag USE_SSH_TUNNEL is True; otherwise, it uses the direct database host and port.
if getattr(DatabaseConfig, "USE_SSH_TUNNEL", False):
# The variable “sshTunnel” opens a forwarder that connects the local port to the remote MySQL host through SSH.
self.sshTunnel = SSHTunnelForwarder(
(DatabaseConfig.SSH_HOST, DatabaseConfig.SSH_PORT),
ssh_username=DatabaseConfig.SSH_USERNAME,
......@@ -36,36 +31,46 @@ class MysqlManager:
remote_bind_address=(self.dbConfig["host"], self.dbConfig["port"]),
)
self.sshTunnel.start()
# The variables “db_host” and “db_port” point to the locally forwarded endpoint when the tunnel is active.
db_host = "127.0.0.1"
db_port = self.sshTunnel.local_bind_port
else:
# The variable “sshTunnel” is set to None when no tunnel is required.
self.sshTunnel = None
# The variables “db_host” and “db_port” point to the remote MySQL server when no tunnel is used.
db_host = self.dbConfig["host"]
db_port = self.dbConfig["port"]
# ───────────────────────────────────────────────────────
# SQLAlchemy‑Engine mit QueuePool & Timeouts erstellen
# ───────────────────────────────────────────────────────
# The variable “engine” creates an SQLAlchemy engine that uses a QueuePool and applies timeout settings taken from the configuration.
self.engine = create_engine(
f"mysql+pymysql://{self.dbConfig['user']}:{self.dbConfig['password']}@{db_host}:{db_port}/{self.dbConfig['database']}",
echo=False,
poolclass=QueuePool,
pool_size=getattr(DatabaseConfig, "POOL_SIZE", 1),
max_overflow=getattr(DatabaseConfig, "MAX_OVERFLOW", 0),
pool_recycle=getattr(DatabaseConfig, "POOL_RECYCLE", 3600), # Sekunden
pool_recycle=getattr(DatabaseConfig, "POOL_RECYCLE", 3600),
pool_pre_ping=True,
connect_args={"connect_timeout": getattr(DatabaseConfig, "CONNECT_TIMEOUT", 30)},
)
# Session Factory sofort initialisieren
# The variable “dbSession” stores the first session instance created from the session factory bound to the engine.
self.dbSession = sessionmaker(bind=self.engine)()
# Gibt die aktuelle Session zurück
# This method returns the current SQLAlchemy session so callers can interact with the database.
def getSession(self):
# The session instance is returned without creating a new one.
return self.dbSession
# Schließt Session und SSH‑Tunnel (falls vorhanden)
# This method closes the current session and shuts down the SSH tunnel when it was created.
def close(self):
# The session is closed to release database resources.
self.dbSession.close()
# This conditional branch stops the SSH tunnel when it exists.
if self.sshTunnel:
self.sshTunnel.stop()
"""
Hauptanwendung (Manager-Kontext)
Startet die Flask-App, erzeugt einen WebManager und registriert
alle Blueprints zentral.
**Neu**
Alle Endpunkte verlangen jetzt zwingend den Query-Parameter
?token=12345
Fehlt der Parameter oder stimmt der Wert nicht, erhält der Client
HTTP/1.1 401 Unauthorized
{"status": "NOK", "message": "Ungültiger oder fehlender Token."}
"""
from __future__ import annotations
import sys
sys.path.append("..") # Projekt-Root im Suchpfad registrieren
import sys; sys.path.append("..")
from flask import Flask, request, jsonify
from manager.MysqlManager import MysqlManager
from models.token_toke import TokenToke
from routes.HealtCheckRouter import blueprint as health_router
from routes.BaseRouter import blueprint as tarifs_router
from routes.EeccxRouter import blueprint as eeccx_router
from routes.BaseRouter import blueprint as tarifs_router
from routes.EeccxRouter import blueprint as eeccx_router
# --------------------------------------------------------------------------- #
# WebManager: registriert sämtliche Blueprints
# --------------------------------------------------------------------------- #
# This class bundles blueprint registration so that all route collections are attached to the Flask application.
class WebManager:
"""Registriert Blueprints und bündelt weitere Infrastruktur."""
# The constructor assigns the provided Flask instance and calls the private registration helper.
def __init__(self, app: Flask) -> None:
self.app = app
self._register_blueprints()
# This helper method iterates over all blueprints and registers each of them on the Flask application.
def _register_blueprints(self) -> None:
"""Alle Blueprint-Objekte an der App anmelden."""
for bp in (health_router, tarifs_router, eeccx_router):
self.app.register_blueprint(bp)
# --------------------------------------------------------------------------- #
# App-Instanz & globale Token-Prüfung
# --------------------------------------------------------------------------- #
TOKEN_VALUE = "12345" # Erlaubter Token-Wert
# A new Flask application instance is created and handed to the WebManager for blueprint registration.
app = Flask(__name__)
WebManager(app)
# This handler executes before every request to enforce the compulsory token parameter and validate it against the database.
@app.before_request
def _require_token():
"""
Globale Pre-Request-Hook:
Schlägt fehl, wenn der Query-Parameter ?token=12345
nicht exakt vorhanden ist.
"""
# This branch allows requests for static files to proceed without token validation.
if request.endpoint == "static":
# Flask-static-Files nicht schützen
return None
# The variable “token” stores the value of the ?token query parameter or None when absent.
token = request.args.get("token")
if token != TOKEN_VALUE:
# This branch rejects the request when the token parameter is missing.
if not token:
return (
jsonify({"message": "Please enter a valid token."}),
jsonify({"status": "NOAUTH", "message": "Please enter a valid token."}),
401,
)
# The variable “session” holds a new SQLAlchemy session obtained from the MysqlManager.
session = MysqlManager().getSession()
try:
# The variable “token_exists” evaluates to True when a matching token record is found in the database.
token_exists = (
session.query(TokenToke)
.filter_by(token_toke=token)
.first()
is not None
)
finally:
session.close()
# This branch rejects the request when the supplied token does not exist in the database.
if not token_exists:
return (
jsonify({"status": "NOAUTH", "message": "Please enter a valid token."}),
401,
)
# --------------------------------------------------------------------------- #
# Startpunkt
# --------------------------------------------------------------------------- #
# The application starts on all network interfaces on port 80 when the module is executed directly.
if __name__ == "__main__":
# Server auf allen Interfaces, Port 80 starten
app.run(host="0.0.0.0", port=80)
from sqlalchemy import Column, Integer, String, DateTime
from models._system import Base
class TokenToke(Base):
__tablename__ = 'token_toke'
id_toke = Column(
Integer,
primary_key=True,
autoincrement=True
)
token_toke = Column(
String(255),
nullable=False
)
owner_toke = Column(
String(255),
nullable=False
)
created_toke = Column(
DateTime,
nullable=False
)
# MAUI Data Toolkit
## Tutorials
....
....
....
## JupyterLab
To further develop or test this project use jupyter lab. Take care that "notebooks" are only concepts with your pc as python environment. Production code runs only inside the docker environment (stored in e.g. "commands" or "manager" folder).
......@@ -8,31 +13,18 @@ To further develop or test this project use jupyter lab. Take care that "noteboo
jupyter lab
```
## Docker & ECR
## Docker
Use Docker to deploy this package in a production environment. Log in to Amazon ECR with the AWS CLI:
```bash
aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin ???
```
Build, tag, and push the image:
Use Docker to build this package for a production environment.
```bash
docker build --platform linux/amd64 -t maui:latest .
docker tag maui:latest ???
docker push ???
```
To pull and run the container, use:
```bash
docker pull ???
docker run -it -d --restart always -p 80:80 ???
docker run -it -d --restart always -p 80:80 maui:latest
```
Alternatively, for local development with mounted volumes:
```bash
docker run -it \
-v ./commands:/maui/commands \
......
This diff is collapsed.
This diff is collapsed.
"""
Health-Check-Router
Kapselt den Endpunkt / für den System-Gesundheitscheck.
"""
from flask import Blueprint, jsonify
# Blueprint-Name = Dateiname ohne Punkte; verhindert ValueError
# The blueprint instance is named after the current module without dots to avoid a ValueError on registration.
blueprint = Blueprint(__name__.rsplit(".", 1)[-1], __name__)
# This function handles HTTP GET requests to the root path and returns a simple health-check response.
@blueprint.route("/", methods=["GET"])
def index():
"""
GET /
Liefert einen einfachen JSON-Status.
"""
return jsonify({"message": "The API is working."})
# The function returns a JSON object indicating that the API is operational.
return jsonify({"message": "The API is working.", "status": "OK"})
No preview for this file type
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment