Skip to content

C
crawler
Commit 38ae6ecf authored by Marco Schmiedel's avatar Marco Schmiedel
Browse files
Options

fix

parent da9e7246
Hide whitespace changes
Inline Side-by-side
Showing with 953 additions and 3179 deletions
notebooks/CalculateDetailsFromPdfWithGpt.ipynb
View file @ 38ae6ecf
...@@ -15,52 +15,403 @@ ...@@ -15,52 +15,403 @@
"INFO: Suche nach PDF-Dateien in '../cache' …\n", "INFO: Suche nach PDF-Dateien in '../cache' …\n",
"\n", "\n",
"--- Verarbeitung ID: 3243715 ---\n", "--- Verarbeitung ID: 3243715 ---\n",
"INFO: GPT-Abfrage Versuch 1/3 …\n", "INFO: details_base bereits gefüllt (id_base=219).\n",
"INFO: JSON in id_base 87 gespeichert.\n",
"\n", "\n",
"--- Verarbeitung ID: 3332926 ---\n", "--- Verarbeitung ID: 3332926 ---\n",
"INFO: details_base bereits gefüllt (id_base=96).\n", "INFO: details_base bereits gefüllt (id_base=228).\n",
"\n", "\n",
"--- Verarbeitung ID: 3337091 ---\n", "--- Verarbeitung ID: 3337091 ---\n",
"INFO: details_base bereits gefüllt (id_base=103).\n", "INFO: details_base bereits gefüllt (id_base=235).\n",
"\n", "\n",
"--- Verarbeitung ID: 3337095 ---\n", "--- Verarbeitung ID: 3337095 ---\n",
"INFO: GPT-Abfrage Versuch 1/3 …\n", "INFO: details_base bereits gefüllt (id_base=233).\n",
"INFO: JSON in id_base 101 gespeichert.\n",
"\n", "\n",
"--- Verarbeitung ID: 3380690 ---\n", "--- Verarbeitung ID: 3380690 ---\n",
"INFO: GPT-Abfrage Versuch 1/3 …\n", "INFO: details_base bereits gefüllt (id_base=209).\n",
"INFO: JSON in id_base 77 gespeichert.\n",
"\n", "\n",
"--- Verarbeitung ID: 3385056 ---\n", "--- Verarbeitung ID: 3385056 ---\n",
"INFO: GPT-Abfrage Versuch 1/3 …\n" "INFO: details_base bereits gefüllt (id_base=215).\n",
] "\n",
}, "--- Verarbeitung ID: 3385059 ---\n",
{ "INFO: details_base bereits gefüllt (id_base=214).\n",
"ename": "KeyboardInterrupt", "\n",
"evalue": "", "--- Verarbeitung ID: 3387917 ---\n",
"output_type": "error", "INFO: details_base bereits gefüllt (id_base=218).\n",
"traceback": [ "\n",
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "--- Verarbeitung ID: 3388975 ---\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "INFO: details_base bereits gefüllt (id_base=207).\n",
"Cell \u001b[0;32mIn[1], line 230\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mINFO: GPT-Abfrage Versuch \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mattempt\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/3 …\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 229\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 230\u001b[0m raw \u001b[38;5;241m=\u001b[39m \u001b[43mgpt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchat\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfull_prompt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgpt-4o-mini\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 231\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 232\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFEHLER: GPT-Abfrage fehlgeschlagen: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", "\n",
"File \u001b[0;32m~/Local/Development/Maui/notebooks/../manager/OpenAiManager.py:42\u001b[0m, in \u001b[0;36mOpenAiManager.chat\u001b[0;34m(self, prompt, model)\u001b[0m\n\u001b[1;32m 36\u001b[0m data \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 37\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m: model,\n\u001b[1;32m 38\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessages\u001b[39m\u001b[38;5;124m\"\u001b[39m: [{\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrole\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124muser\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m\"\u001b[39m: prompt}]\n\u001b[1;32m 39\u001b[0m }\n\u001b[1;32m 41\u001b[0m \u001b[38;5;66;03m# An dieser Stelle wird die Anfrage an die OpenAI-API gesendet.\u001b[39;00m\n\u001b[0;32m---> 42\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;66;03m# Die Antwort wird aus dem JSON-Format in ein Python-Objekt konvertiert.\u001b[39;00m\n\u001b[1;32m 45\u001b[0m result \u001b[38;5;241m=\u001b[39m response\u001b[38;5;241m.\u001b[39mjson()\n", "--- Verarbeitung ID: 3389002 ---\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/requests/api.py:115\u001b[0m, in \u001b[0;36mpost\u001b[0;34m(url, data, json, **kwargs)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(url, data\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, json\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 104\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[1;32m 105\u001b[0m \n\u001b[1;32m 106\u001b[0m \u001b[38;5;124;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;124;03m :rtype: requests.Response\u001b[39;00m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpost\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "INFO: details_base bereits gefüllt (id_base=208).\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m 56\u001b[0m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m 58\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m sessions\u001b[38;5;241m.\u001b[39mSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 585\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m 586\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m 587\u001b[0m }\n\u001b[1;32m 588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", "--- Verarbeitung ID: 3398803 ---\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m 700\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m 702\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 705\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m 706\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n", "INFO: details_base bereits gefüllt (id_base=155).\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/requests/adapters.py:667\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 664\u001b[0m timeout \u001b[38;5;241m=\u001b[39m TimeoutSauce(connect\u001b[38;5;241m=\u001b[39mtimeout, read\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m 666\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 667\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 668\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 669\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 670\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 671\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 677\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 678\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 679\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 681\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m 682\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(err, request\u001b[38;5;241m=\u001b[39mrequest)\n", "\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/urllib3/connectionpool.py:703\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prepare_proxy(conn)\n\u001b[1;32m 702\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 704\u001b[0m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 705\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 706\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 707\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 708\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 709\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 710\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 711\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 713\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m 714\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m 715\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m 716\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n\u001b[1;32m 717\u001b[0m response_conn \u001b[38;5;241m=\u001b[39m conn \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m release_conn \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", "--- Verarbeitung ID: 3398807 ---\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/urllib3/connectionpool.py:449\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m 444\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39mgetresponse()\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 446\u001b[0m \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m 447\u001b[0m \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m 448\u001b[0m \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[0;32m--> 449\u001b[0m \u001b[43msix\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_from\u001b[49m\u001b[43m(\u001b[49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 450\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 451\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_raise_timeout(err\u001b[38;5;241m=\u001b[39me, url\u001b[38;5;241m=\u001b[39murl, timeout_value\u001b[38;5;241m=\u001b[39mread_timeout)\n", "INFO: details_base bereits gefüllt (id_base=153).\n",
"File \u001b[0;32m<string>:3\u001b[0m, in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n", "\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/urllib3/connectionpool.py:444\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m 441\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[1;32m 442\u001b[0m \u001b[38;5;66;03m# Python 3\u001b[39;00m\n\u001b[1;32m 443\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 444\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 446\u001b[0m \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m 447\u001b[0m \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m 448\u001b[0m \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[1;32m 449\u001b[0m six\u001b[38;5;241m.\u001b[39mraise_from(e, \u001b[38;5;28;01mNone\u001b[39;00m)\n", "--- Verarbeitung ID: 3398811 ---\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1374\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1372\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1373\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1374\u001b[0m \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1375\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n\u001b[1;32m 1376\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n", "INFO: details_base bereits gefüllt (id_base=151).\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:318\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 316\u001b[0m \u001b[38;5;66;03m# read until we get a non-100 response\u001b[39;00m\n\u001b[1;32m 317\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 318\u001b[0m version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 319\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n\u001b[1;32m 320\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n", "\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:279\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 278\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 279\u001b[0m line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miso-8859-1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 280\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n\u001b[1;32m 281\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstatus line\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", "--- Verarbeitung ID: 3398815 ---\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/socket.py:705\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 703\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m 704\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 705\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 706\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m 707\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", "INFO: details_base bereits gefüllt (id_base=157).\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/ssl.py:1274\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m 1270\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 1271\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 1272\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m 1273\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1274\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1275\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1276\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv_into(buffer, nbytes, flags)\n", "\n",
"File \u001b[0;32m~/anaconda3/lib/python3.10/ssl.py:1130\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m 1128\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1129\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1131\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1132\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m)\n", "--- Verarbeitung ID: 3415206 ---\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: " "INFO: details_base bereits gefüllt (id_base=212).\n",
"\n",
"--- Verarbeitung ID: 3430251 ---\n",
"INFO: details_base bereits gefüllt (id_base=234).\n",
"\n",
"--- Verarbeitung ID: 3435067 ---\n",
"INFO: details_base bereits gefüllt (id_base=156).\n",
"\n",
"--- Verarbeitung ID: 3435071 ---\n",
"INFO: details_base bereits gefüllt (id_base=154).\n",
"\n",
"--- Verarbeitung ID: 3435075 ---\n",
"INFO: details_base bereits gefüllt (id_base=152).\n",
"\n",
"--- Verarbeitung ID: 3435079 ---\n",
"INFO: details_base bereits gefüllt (id_base=158).\n",
"\n",
"--- Verarbeitung ID: 3446028 ---\n",
"INFO: details_base bereits gefüllt (id_base=166).\n",
"\n",
"--- Verarbeitung ID: 3448928 ---\n",
"INFO: details_base bereits gefüllt (id_base=232).\n",
"\n",
"--- Verarbeitung ID: 3461557 ---\n",
"INFO: details_base bereits gefüllt (id_base=164).\n",
"\n",
"--- Verarbeitung ID: 3461561 ---\n",
"INFO: details_base bereits gefüllt (id_base=165).\n",
"\n",
"--- Verarbeitung ID: 3461565 ---\n",
"INFO: details_base bereits gefüllt (id_base=167).\n",
"\n",
"--- Verarbeitung ID: 3461569 ---\n",
"INFO: details_base bereits gefüllt (id_base=168).\n",
"\n",
"--- Verarbeitung ID: 3473445 ---\n",
"INFO: details_base bereits gefüllt (id_base=213).\n",
"\n",
"--- Verarbeitung ID: 3476568 ---\n",
"INFO: details_base bereits gefüllt (id_base=199).\n",
"\n",
"--- Verarbeitung ID: 3596218 ---\n",
"INFO: details_base bereits gefüllt (id_base=160).\n",
"\n",
"--- Verarbeitung ID: 3649995 ---\n",
"INFO: details_base bereits gefüllt (id_base=159).\n",
"\n",
"--- Verarbeitung ID: 3650007 ---\n",
"INFO: details_base bereits gefüllt (id_base=161).\n",
"\n",
"--- Verarbeitung ID: 3650019 ---\n",
"INFO: details_base bereits gefüllt (id_base=162).\n",
"\n",
"--- Verarbeitung ID: 3650031 ---\n",
"INFO: details_base bereits gefüllt (id_base=163).\n",
"\n",
"--- Verarbeitung ID: 3697996 ---\n",
"INFO: details_base bereits gefüllt (id_base=200).\n",
"\n",
"--- Verarbeitung ID: 3711205 ---\n",
"INFO: details_base bereits gefüllt (id_base=202).\n",
"\n",
"--- Verarbeitung ID: 3711328 ---\n",
"INFO: details_base bereits gefüllt (id_base=204).\n",
"\n",
"--- Verarbeitung ID: 3743289 ---\n",
"INFO: details_base bereits gefüllt (id_base=210).\n",
"\n",
"--- Verarbeitung ID: 3743301 ---\n",
"INFO: details_base bereits gefüllt (id_base=211).\n",
"\n",
"--- Verarbeitung ID: 3750083 ---\n",
"INFO: details_base bereits gefüllt (id_base=243).\n",
"\n",
"--- Verarbeitung ID: 3750710 ---\n",
"INFO: details_base bereits gefüllt (id_base=242).\n",
"\n",
"--- Verarbeitung ID: 3750758 ---\n",
"INFO: details_base bereits gefüllt (id_base=241).\n",
"\n",
"--- Verarbeitung ID: 3750806 ---\n",
"INFO: details_base bereits gefüllt (id_base=244).\n",
"\n",
"--- Verarbeitung ID: 3765006 ---\n",
"INFO: details_base bereits gefüllt (id_base=169).\n",
"\n",
"--- Verarbeitung ID: 3782151 ---\n",
"INFO: details_base bereits gefüllt (id_base=170).\n",
"\n",
"--- Verarbeitung ID: 3782322 ---\n",
"INFO: details_base bereits gefüllt (id_base=258).\n",
"\n",
"--- Verarbeitung ID: 3782325 ---\n",
"INFO: details_base bereits gefüllt (id_base=259).\n",
"\n",
"--- Verarbeitung ID: 3782328 ---\n",
"INFO: details_base bereits gefüllt (id_base=261).\n",
"\n",
"--- Verarbeitung ID: 3782331 ---\n",
"INFO: details_base bereits gefüllt (id_base=260).\n",
"\n",
"--- Verarbeitung ID: 3833020 ---\n",
"INFO: details_base bereits gefüllt (id_base=221).\n",
"\n",
"--- Verarbeitung ID: 3833026 ---\n",
"INFO: details_base bereits gefüllt (id_base=220).\n",
"\n",
"--- Verarbeitung ID: 3833773 ---\n",
"INFO: details_base bereits gefüllt (id_base=198).\n",
"\n",
"--- Verarbeitung ID: 3833776 ---\n",
"INFO: details_base bereits gefüllt (id_base=197).\n",
"\n",
"--- Verarbeitung ID: 3833953 ---\n",
"INFO: details_base bereits gefüllt (id_base=191).\n",
"\n",
"--- Verarbeitung ID: 3833956 ---\n",
"INFO: details_base bereits gefüllt (id_base=193).\n",
"\n",
"--- Verarbeitung ID: 3833959 ---\n",
"INFO: details_base bereits gefüllt (id_base=192).\n",
"\n",
"--- Verarbeitung ID: 3833962 ---\n",
"INFO: details_base bereits gefüllt (id_base=171).\n",
"\n",
"--- Verarbeitung ID: 3833965 ---\n",
"INFO: details_base bereits gefüllt (id_base=173).\n",
"\n",
"--- Verarbeitung ID: 3833968 ---\n",
"INFO: details_base bereits gefüllt (id_base=172).\n",
"\n",
"--- Verarbeitung ID: 3833971 ---\n",
"INFO: details_base bereits gefüllt (id_base=176).\n",
"\n",
"--- Verarbeitung ID: 3833974 ---\n",
"INFO: details_base bereits gefüllt (id_base=178).\n",
"\n",
"--- Verarbeitung ID: 3833977 ---\n",
"INFO: details_base bereits gefüllt (id_base=177).\n",
"\n",
"--- Verarbeitung ID: 3833983 ---\n",
"INFO: details_base bereits gefüllt (id_base=183).\n",
"\n",
"--- Verarbeitung ID: 3833986 ---\n",
"INFO: details_base bereits gefüllt (id_base=185).\n",
"\n",
"--- Verarbeitung ID: 3833989 ---\n",
"INFO: details_base bereits gefüllt (id_base=184).\n",
"\n",
"--- Verarbeitung ID: 3833995 ---\n",
"INFO: details_base bereits gefüllt (id_base=188).\n",
"\n",
"--- Verarbeitung ID: 3833998 ---\n",
"INFO: details_base bereits gefüllt (id_base=190).\n",
"\n",
"--- Verarbeitung ID: 3834001 ---\n",
"INFO: details_base bereits gefüllt (id_base=189).\n",
"\n",
"--- Verarbeitung ID: 3861200 ---\n",
"INFO: details_base bereits gefüllt (id_base=216).\n",
"\n",
"--- Verarbeitung ID: 3861206 ---\n",
"INFO: details_base bereits gefüllt (id_base=217).\n",
"\n",
"--- Verarbeitung ID: 3877175 ---\n",
"INFO: details_base bereits gefüllt (id_base=201).\n",
"\n",
"--- Verarbeitung ID: 3877193 ---\n",
"INFO: details_base bereits gefüllt (id_base=203).\n",
"\n",
"--- Verarbeitung ID: 3877208 ---\n",
"INFO: details_base bereits gefüllt (id_base=206).\n",
"\n",
"--- Verarbeitung ID: 3877211 ---\n",
"INFO: details_base bereits gefüllt (id_base=205).\n",
"\n",
"--- Verarbeitung ID: 3877289 ---\n",
"INFO: details_base bereits gefüllt (id_base=145).\n",
"\n",
"--- Verarbeitung ID: 3877301 ---\n",
"INFO: details_base bereits gefüllt (id_base=147).\n",
"\n",
"--- Verarbeitung ID: 3877313 ---\n",
"INFO: details_base bereits gefüllt (id_base=146).\n",
"\n",
"--- Verarbeitung ID: 3877325 ---\n",
"INFO: details_base bereits gefüllt (id_base=133).\n",
"\n",
"--- Verarbeitung ID: 3877337 ---\n",
"INFO: details_base bereits gefüllt (id_base=135).\n",
"\n",
"--- Verarbeitung ID: 3877349 ---\n",
"INFO: details_base bereits gefüllt (id_base=134).\n",
"\n",
"--- Verarbeitung ID: 3878165 ---\n",
"INFO: details_base bereits gefüllt (id_base=139).\n",
"\n",
"--- Verarbeitung ID: 3878177 ---\n",
"INFO: details_base bereits gefüllt (id_base=141).\n",
"\n",
"--- Verarbeitung ID: 3878189 ---\n",
"INFO: details_base bereits gefüllt (id_base=140).\n",
"\n",
"--- Verarbeitung ID: 3878213 ---\n",
"INFO: details_base bereits gefüllt (id_base=136).\n",
"\n",
"--- Verarbeitung ID: 3878225 ---\n",
"INFO: details_base bereits gefüllt (id_base=138).\n",
"\n",
"--- Verarbeitung ID: 3878237 ---\n",
"INFO: details_base bereits gefüllt (id_base=137).\n",
"\n",
"--- Verarbeitung ID: 3878261 ---\n",
"INFO: details_base bereits gefüllt (id_base=148).\n",
"\n",
"--- Verarbeitung ID: 3878273 ---\n",
"INFO: details_base bereits gefüllt (id_base=150).\n",
"\n",
"--- Verarbeitung ID: 3878285 ---\n",
"INFO: details_base bereits gefüllt (id_base=149).\n",
"\n",
"--- Verarbeitung ID: 3905336 ---\n",
"INFO: details_base bereits gefüllt (id_base=142).\n",
"\n",
"--- Verarbeitung ID: 3905348 ---\n",
"INFO: details_base bereits gefüllt (id_base=144).\n",
"\n",
"--- Verarbeitung ID: 3905360 ---\n",
"INFO: details_base bereits gefüllt (id_base=143).\n",
"\n",
"--- Verarbeitung ID: 3907523 ---\n",
"INFO: details_base bereits gefüllt (id_base=227).\n",
"\n",
"--- Verarbeitung ID: 3907559 ---\n",
"INFO: details_base bereits gefüllt (id_base=225).\n",
"\n",
"--- Verarbeitung ID: 3907595 ---\n",
"INFO: details_base bereits gefüllt (id_base=223).\n",
"\n",
"--- Verarbeitung ID: 3907631 ---\n",
"INFO: details_base bereits gefüllt (id_base=230).\n",
"\n",
"--- Verarbeitung ID: 3935061 ---\n",
"INFO: details_base bereits gefüllt (id_base=257).\n",
"\n",
"--- Verarbeitung ID: 3936264 ---\n",
"INFO: details_base bereits gefüllt (id_base=240).\n",
"\n",
"--- Verarbeitung ID: 3936300 ---\n",
"INFO: details_base bereits gefüllt (id_base=238).\n",
"\n",
"--- Verarbeitung ID: 3936336 ---\n",
"INFO: details_base bereits gefüllt (id_base=237).\n",
"\n",
"--- Verarbeitung ID: 3936372 ---\n",
"INFO: details_base bereits gefüllt (id_base=236).\n",
"\n",
"--- Verarbeitung ID: 3936408 ---\n",
"INFO: details_base bereits gefüllt (id_base=239).\n",
"\n",
"--- Verarbeitung ID: 3973250 ---\n",
"INFO: details_base bereits gefüllt (id_base=254).\n",
"\n",
"--- Verarbeitung ID: 3973253 ---\n",
"INFO: details_base bereits gefüllt (id_base=256).\n",
"\n",
"--- Verarbeitung ID: 3973256 ---\n",
"INFO: details_base bereits gefüllt (id_base=255).\n",
"\n",
"--- Verarbeitung ID: 3973259 ---\n",
"INFO: details_base bereits gefüllt (id_base=251).\n",
"\n",
"--- Verarbeitung ID: 3973262 ---\n",
"INFO: details_base bereits gefüllt (id_base=253).\n",
"\n",
"--- Verarbeitung ID: 3973265 ---\n",
"INFO: details_base bereits gefüllt (id_base=252).\n",
"\n",
"--- Verarbeitung ID: 3973277 ---\n",
"INFO: details_base bereits gefüllt (id_base=248).\n",
"\n",
"--- Verarbeitung ID: 3973280 ---\n",
"INFO: details_base bereits gefüllt (id_base=250).\n",
"\n",
"--- Verarbeitung ID: 3973283 ---\n",
"INFO: details_base bereits gefüllt (id_base=249).\n",
"\n",
"--- Verarbeitung ID: 3973286 ---\n",
"INFO: details_base bereits gefüllt (id_base=245).\n",
"\n",
"--- Verarbeitung ID: 3973289 ---\n",
"INFO: details_base bereits gefüllt (id_base=247).\n",
"\n",
"--- Verarbeitung ID: 3973292 ---\n",
"INFO: details_base bereits gefüllt (id_base=246).\n",
"\n",
"--- Verarbeitung ID: 3973295 ---\n",
"INFO: details_base bereits gefüllt (id_base=262).\n",
"\n",
"--- Verarbeitung ID: 3973298 ---\n",
"INFO: details_base bereits gefüllt (id_base=264).\n",
"\n",
"--- Verarbeitung ID: 3973301 ---\n",
"INFO: details_base bereits gefüllt (id_base=263).\n",
"\n",
"--- Verarbeitung ID: 3974111 ---\n",
"INFO: details_base bereits gefüllt (id_base=196).\n",
"\n",
"--- Verarbeitung ID: 3975845 ---\n",
"INFO: details_base bereits gefüllt (id_base=194).\n",
"\n",
"--- Verarbeitung ID: 3975848 ---\n",
"INFO: details_base bereits gefüllt (id_base=174).\n",
"\n",
"--- Verarbeitung ID: 3975851 ---\n",
"INFO: details_base bereits gefüllt (id_base=179).\n",
"\n",
"--- Verarbeitung ID: 3975854 ---\n",
"INFO: details_base bereits gefüllt (id_base=181).\n",
"\n",
"--- Verarbeitung ID: 3975860 ---\n",
"INFO: details_base bereits gefüllt (id_base=186).\n",
"\n",
"--- Verarbeitung ID: 3975866 ---\n",
"INFO: details_base bereits gefüllt (id_base=195).\n",
"\n",
"--- Verarbeitung ID: 3975869 ---\n",
"INFO: details_base bereits gefüllt (id_base=175).\n",
"\n",
"--- Verarbeitung ID: 3975872 ---\n",
"INFO: details_base bereits gefüllt (id_base=180).\n",
"\n",
"--- Verarbeitung ID: 3975875 ---\n",
"INFO: details_base bereits gefüllt (id_base=182).\n",
"\n",
"--- Verarbeitung ID: 3975881 ---\n",
"INFO: details_base bereits gefüllt (id_base=187).\n",
"\n",
"--- Verarbeitung ID: 4069662 ---\n",
"INFO: details_base bereits gefüllt (id_base=231).\n",
"\n",
"--- Verarbeitung ID: 4069698 ---\n",
"INFO: details_base bereits gefüllt (id_base=226).\n",
"\n",
"--- Verarbeitung ID: 4069734 ---\n",
"INFO: details_base bereits gefüllt (id_base=224).\n",
"\n",
"--- Verarbeitung ID: 4069770 ---\n",
"INFO: GPT-Abfrage Versuch 1/3 …\n",
"INFO: JSON in id_base 222 gespeichert.\n",
"\n",
"--- Verarbeitung ID: 4069806 ---\n",
"INFO: GPT-Abfrage Versuch 1/3 …\n",
"INFO: JSON in id_base 229 gespeichert.\n",
"INFO: Verarbeitung abgeschlossen.\n"
] ]
} }
], ],
...@@ -167,7 +518,7 @@ ...@@ -167,7 +518,7 @@
"\n", "\n",
" # In dieser Abzweigung wird geprüft, ob die Datei existiert.\n", " # In dieser Abzweigung wird geprüft, ob die Datei existiert.\n",
" if not os.path.exists(pdfPath):\n", " if not os.path.exists(pdfPath):\n",
" print(f\" INFO: Datei nicht gefunden: {os.path.basename(pdfPath)}\")\n", " print(f\"INFO: Datei nicht gefunden: {os.path.basename(pdfPath)}\")\n",
" return None\n", " return None\n",
"\n", "\n",
" # In dieser Liste wird der Text seitenweise abgelegt.\n", " # In dieser Liste wird der Text seitenweise abgelegt.\n",
...@@ -184,7 +535,7 @@ ...@@ -184,7 +535,7 @@
"\n", "\n",
" # In dieser Abzweigung wird eine Meldung ausgegeben, wenn kein Text gefunden wurde.\n", " # In dieser Abzweigung wird eine Meldung ausgegeben, wenn kein Text gefunden wurde.\n",
" if not pageList:\n", " if not pageList:\n",
" print(f\" INFO: Kein Text in {os.path.basename(pdfPath)}\")\n", " print(f\"INFO: Kein Text in {os.path.basename(pdfPath)}\")\n",
" return None\n", " return None\n",
"\n", "\n",
" # Hier wird der zusammengesetzte Text zurückgegeben.\n", " # Hier wird der zusammengesetzte Text zurückgegeben.\n",
...@@ -192,12 +543,12 @@ ...@@ -192,12 +543,12 @@
"\n", "\n",
" # In dieser Abzweigung wird ein spezieller Fehler von pypdf abgefangen.\n", " # In dieser Abzweigung wird ein spezieller Fehler von pypdf abgefangen.\n",
" except PdfReadError as e:\n", " except PdfReadError as e:\n",
" print(f\" WARNUNG: pypdf konnte '{os.path.basename(pdfPath)}' nicht lesen: {e}\")\n", " print(f\"WARNUNG: pypdf konnte '{os.path.basename(pdfPath)}' nicht lesen: {e}\")\n",
" return None\n", " return None\n",
"\n", "\n",
" # In dieser Abzweigung werden andere Fehler behandelt.\n", " # In dieser Abzweigung werden andere Fehler behandelt.\n",
" except Exception:\n", " except Exception:\n",
" print(f\" FEHLER: Unerwarteter Fehler bei '{os.path.basename(pdfPath)}':\")\n", " print(f\"FEHLER: Unerwarteter Fehler bei '{os.path.basename(pdfPath)}':\")\n",
" traceback.print_exc(limit=1)\n", " traceback.print_exc(limit=1)\n",
" return None\n", " return None\n",
"\n", "\n",
...@@ -253,12 +604,12 @@ ...@@ -253,12 +604,12 @@
"\n", "\n",
" # In dieser Abzweigung wird geprüft, ob eine Konvertierung möglich war.\n", " # In dieser Abzweigung wird geprüft, ob eine Konvertierung möglich war.\n",
" if data is None:\n", " if data is None:\n",
" print(\" VALIDATION: Antwort ist kein gültiges/rekonstruierbares JSON.\")\n", " print(\"VALIDATION: Antwort ist kein gültiges/rekonstruierbares JSON.\")\n",
" return False, None\n", " return False, None\n",
"\n", "\n",
" # In dieser Abzweigung wird geprüft, ob das geladene Objekt ein Dictionary ist.\n", " # In dieser Abzweigung wird geprüft, ob das geladene Objekt ein Dictionary ist.\n",
" if not isinstance(data, dict):\n", " if not isinstance(data, dict):\n",
" print(\" VALIDATION: JSON ist kein Objekt.\")\n", " print(\"VALIDATION: JSON ist kein Objekt.\")\n",
" return False, None\n", " return False, None\n",
"\n", "\n",
" # Hier wird eine Liste der fehlenden Keys erstellt.\n", " # Hier wird eine Liste der fehlenden Keys erstellt.\n",
...@@ -266,7 +617,7 @@ ...@@ -266,7 +617,7 @@
"\n", "\n",
" # In dieser Abzweigung wird eine Meldung ausgegeben, falls Keys fehlen.\n", " # In dieser Abzweigung wird eine Meldung ausgegeben, falls Keys fehlen.\n",
" if missingKeys:\n", " if missingKeys:\n",
" print(f\" VALIDATION: Fehlende Schlüssel: {', '.join(missingKeys)}\")\n", " print(f\"VALIDATION: Fehlende Schlüssel: {', '.join(missingKeys)}\")\n",
" return False, None\n", " return False, None\n",
"\n", "\n",
" # Hier wird das Ergebnis zurückgegeben, wenn alles stimmt.\n", " # Hier wird das Ergebnis zurückgegeben, wenn alles stimmt.\n",
...@@ -356,7 +707,7 @@ ...@@ -356,7 +707,7 @@
"\n", "\n",
" # In diesem try-Block wird die GPT-Anfrage durchgeführt.\n", " # In diesem try-Block wird die GPT-Anfrage durchgeführt.\n",
" try:\n", " try:\n",
" rawResponse = gptManager.chat(fullPrompt, model=\"gpt-4o-mini\")\n", " rawResponse = gptManager.chat(fullPrompt, model=\"gpt-4.1\")\n",
"\n", "\n",
" # In dieser Abzweigung wird eine Fehlermeldung ausgegeben, wenn die Abfrage fehlschlägt.\n", " # In dieser Abzweigung wird eine Fehlermeldung ausgegeben, wenn die Abfrage fehlschlägt.\n",
" except Exception as e:\n", " except Exception as e:\n",
...@@ -371,7 +722,7 @@ ...@@ -371,7 +722,7 @@
" validatedPayload = dataResult\n", " validatedPayload = dataResult\n",
" break\n", " break\n",
"\n", "\n",
" print(\" WARNUNG: Antwort ungültig – retry …\")\n", " print(\"WARNUNG: Antwort ungültig – retry …\")\n",
"\n", "\n",
" # In dieser Abzweigung wird eine Meldung ausgegeben, wenn alle Versuche fehlschlagen.\n", " # In dieser Abzweigung wird eine Meldung ausgegeben, wenn alle Versuche fehlschlagen.\n",
" if not validatedPayload:\n", " if not validatedPayload:\n",
......
notebooks/DownloadDataFromMauiWithSelenium.ipynb
View file @ 38ae6ecf
This source diff could not be displayed because it is too large. You can view the blob instead.
notebooks/ImportCsvToDatabase.ipynb
View file @ 38ae6ecf
...@@ -27,17 +27,172 @@ ...@@ -27,17 +27,172 @@
" <thead>\n", " <thead>\n",
" <tr style=\"text-align: right;\">\n", " <tr style=\"text-align: right;\">\n",
" <th></th>\n", " <th></th>\n",
" <th>id_base</th>\n",
" <th>provider_base</th>\n",
" <th>providercode_base</th>\n",
" <th>name_base</th>\n",
" <th>network_base</th>\n",
" <th>created_base</th>\n",
" <th>updated_base</th>\n",
" </tr>\n", " </tr>\n",
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>133</td>\n",
" <td>Freenet | Cellway</td>\n",
" <td>3877325</td>\n",
" <td>Allnet Flat 20 GB Telekom (Okt 2024)</td>\n",
" <td>1</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>134</td>\n",
" <td>Freenet | Cellway</td>\n",
" <td>3877349</td>\n",
" <td>Allnet Flat 20 GB Telekom (Okt 2024) mit Smart...</td>\n",
" <td>1</td>\n",
" <td>2025-04-22 11:44:22</td>\n",
" <td>2025-04-22 11:44:22</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>135</td>\n",
" <td>Freenet | Cellway</td>\n",
" <td>3877337</td>\n",
" <td>Allnet Flat 20 GB Telekom (Okt 2024) mit Smart...</td>\n",
" <td>1</td>\n",
" <td>2025-04-22 11:44:42</td>\n",
" <td>2025-04-22 11:44:42</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>136</td>\n",
" <td>Freenet | Cellway</td>\n",
" <td>3878213</td>\n",
" <td>Allnet Flat 25 GB Telekom (Okt 2024)</td>\n",
" <td>1</td>\n",
" <td>2025-04-22 11:45:04</td>\n",
" <td>2025-04-22 11:45:04</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>137</td>\n",
" <td>Freenet | Cellway</td>\n",
" <td>3878237</td>\n",
" <td>Allnet Flat 25 GB Telekom (Okt 2024) mit Smart...</td>\n",
" <td>1</td>\n",
" <td>2025-04-22 11:45:24</td>\n",
" <td>2025-04-22 11:45:24</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>127</th>\n",
" <td>260</td>\n",
" <td>Freenet | Netzbetreiber</td>\n",
" <td>3782331</td>\n",
" <td>o2 Mobile Unlimited Smart (Jun 2024) mit Smart...</td>\n",
" <td>4</td>\n",
" <td>2025-04-22 12:17:20</td>\n",
" <td>2025-04-22 12:17:20</td>\n",
" </tr>\n",
" <tr>\n",
" <th>128</th>\n",
" <td>261</td>\n",
" <td>Freenet | Netzbetreiber</td>\n",
" <td>3782328</td>\n",
" <td>o2 Mobile Unlimited Smart (Jun 2024) mit Smart...</td>\n",
" <td>4</td>\n",
" <td>2025-04-22 12:17:38</td>\n",
" <td>2025-04-22 12:17:38</td>\n",
" </tr>\n",
" <tr>\n",
" <th>129</th>\n",
" <td>262</td>\n",
" <td>Freenet | Netzbetreiber</td>\n",
" <td>3973295</td>\n",
" <td>o2 Mobile XL (Nov 2024)</td>\n",
" <td>4</td>\n",
" <td>2025-04-22 12:17:55</td>\n",
" <td>2025-04-22 12:17:55</td>\n",
" </tr>\n",
" <tr>\n",
" <th>130</th>\n",
" <td>263</td>\n",
" <td>Freenet | Netzbetreiber</td>\n",
" <td>3973301</td>\n",
" <td>o2 Mobile XL (Nov 2024) mit Smartphone 10</td>\n",
" <td>4</td>\n",
" <td>2025-04-22 12:18:13</td>\n",
" <td>2025-04-22 12:18:13</td>\n",
" </tr>\n",
" <tr>\n",
" <th>131</th>\n",
" <td>264</td>\n",
" <td>Freenet | Netzbetreiber</td>\n",
" <td>3973298</td>\n",
" <td>o2 Mobile XL (Nov 2024) mit Smartphone 5</td>\n",
" <td>4</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"<p>132 rows × 7 columns</p>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
"Empty DataFrame\n", " id_base provider_base providercode_base \\\n",
"Columns: []\n", "0 133 Freenet | Cellway 3877325 \n",
"Index: []" "1 134 Freenet | Cellway 3877349 \n",
"2 135 Freenet | Cellway 3877337 \n",
"3 136 Freenet | Cellway 3878213 \n",
"4 137 Freenet | Cellway 3878237 \n",
".. ... ... ... \n",
"127 260 Freenet | Netzbetreiber 3782331 \n",
"128 261 Freenet | Netzbetreiber 3782328 \n",
"129 262 Freenet | Netzbetreiber 3973295 \n",
"130 263 Freenet | Netzbetreiber 3973301 \n",
"131 264 Freenet | Netzbetreiber 3973298 \n",
"\n",
" name_base network_base \\\n",
"0 Allnet Flat 20 GB Telekom (Okt 2024) 1 \n",
"1 Allnet Flat 20 GB Telekom (Okt 2024) mit Smart... 1 \n",
"2 Allnet Flat 20 GB Telekom (Okt 2024) mit Smart... 1 \n",
"3 Allnet Flat 25 GB Telekom (Okt 2024) 1 \n",
"4 Allnet Flat 25 GB Telekom (Okt 2024) mit Smart... 1 \n",
".. ... ... \n",
"127 o2 Mobile Unlimited Smart (Jun 2024) mit Smart... 4 \n",
"128 o2 Mobile Unlimited Smart (Jun 2024) mit Smart... 4 \n",
"129 o2 Mobile XL (Nov 2024) 4 \n",
"130 o2 Mobile XL (Nov 2024) mit Smartphone 10 4 \n",
"131 o2 Mobile XL (Nov 2024) mit Smartphone 5 4 \n",
"\n",
" created_base updated_base \n",
"0 2025-04-22 11:43:59 2025-04-22 11:43:59 \n",
"1 2025-04-22 11:44:22 2025-04-22 11:44:22 \n",
"2 2025-04-22 11:44:42 2025-04-22 11:44:42 \n",
"3 2025-04-22 11:45:04 2025-04-22 11:45:04 \n",
"4 2025-04-22 11:45:24 2025-04-22 11:45:24 \n",
".. ... ... \n",
"127 2025-04-22 12:17:20 2025-04-22 12:17:20 \n",
"128 2025-04-22 12:17:38 2025-04-22 12:17:38 \n",
"129 2025-04-22 12:17:55 2025-04-22 12:17:55 \n",
"130 2025-04-22 12:18:13 2025-04-22 12:18:13 \n",
"131 2025-04-22 12:18:31 2025-04-22 12:18:31 \n",
"\n",
"[132 rows x 7 columns]"
] ]
}, },
"metadata": {}, "metadata": {},
...@@ -64,17 +219,196 @@ ...@@ -64,17 +219,196 @@
" <thead>\n", " <thead>\n",
" <tr style=\"text-align: right;\">\n", " <tr style=\"text-align: right;\">\n",
" <th></th>\n", " <th></th>\n",
" <th>id_deal</th>\n",
" <th>base_deal</th>\n",
" <th>providercode_deal</th>\n",
" <th>name_deal</th>\n",
" <th>price_deal</th>\n",
" <th>starts_deal</th>\n",
" <th>stops_deal</th>\n",
" <th>created_deal</th>\n",
" <th>updated_deal</th>\n",
" </tr>\n", " </tr>\n",
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>281</td>\n",
" <td>133</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>25.20168</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>282</td>\n",
" <td>133</td>\n",
" <td>A3908081</td>\n",
" <td>170 EUR Sonderbonus</td>\n",
" <td>25.20168</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>283</td>\n",
" <td>133</td>\n",
" <td>A3908084</td>\n",
" <td>24 x 10 EUR Rabatt auf MGP, 0 EUR Vergütungsve...</td>\n",
" <td>25.20168</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>284</td>\n",
" <td>134</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>33.60504</td>\n",
" <td>2025-04-22 11:44:22</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 11:44:22</td>\n",
" <td>2025-04-22 11:44:22</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>285</td>\n",
" <td>134</td>\n",
" <td>A3908081</td>\n",
" <td>170 EUR Sonderbonus</td>\n",
" <td>33.60504</td>\n",
" <td>2025-04-22 11:44:22</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 11:44:22</td>\n",
" <td>2025-04-22 11:44:22</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>275</th>\n",
" <td>556</td>\n",
" <td>262</td>\n",
" <td>A3990011</td>\n",
" <td>210 EUR Sonderbonus</td>\n",
" <td>42.00840</td>\n",
" <td>2025-04-22 12:17:55</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 12:17:55</td>\n",
" <td>2025-04-22 12:17:55</td>\n",
" </tr>\n",
" <tr>\n",
" <th>276</th>\n",
" <td>557</td>\n",
" <td>263</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>50.41176</td>\n",
" <td>2025-04-22 12:18:13</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 12:18:13</td>\n",
" <td>2025-04-22 12:18:13</td>\n",
" </tr>\n",
" <tr>\n",
" <th>277</th>\n",
" <td>558</td>\n",
" <td>263</td>\n",
" <td>A3990011</td>\n",
" <td>210 EUR Sonderbonus</td>\n",
" <td>50.41176</td>\n",
" <td>2025-04-22 12:18:13</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 12:18:13</td>\n",
" <td>2025-04-22 12:18:13</td>\n",
" </tr>\n",
" <tr>\n",
" <th>278</th>\n",
" <td>559</td>\n",
" <td>264</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>46.21008</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" </tr>\n",
" <tr>\n",
" <th>279</th>\n",
" <td>560</td>\n",
" <td>264</td>\n",
" <td>A3990011</td>\n",
" <td>210 EUR Sonderbonus</td>\n",
" <td>46.21008</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"<p>280 rows × 9 columns</p>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
"Empty DataFrame\n", " id_deal base_deal providercode_deal \\\n",
"Columns: []\n", "0 281 133 \n",
"Index: []" "1 282 133 A3908081 \n",
"2 283 133 A3908084 \n",
"3 284 134 \n",
"4 285 134 A3908081 \n",
".. ... ... ... \n",
"275 556 262 A3990011 \n",
"276 557 263 \n",
"277 558 263 A3990011 \n",
"278 559 264 \n",
"279 560 264 A3990011 \n",
"\n",
" name_deal price_deal \\\n",
"0 25.20168 \n",
"1 170 EUR Sonderbonus 25.20168 \n",
"2 24 x 10 EUR Rabatt auf MGP, 0 EUR Vergütungsve... 25.20168 \n",
"3 33.60504 \n",
"4 170 EUR Sonderbonus 33.60504 \n",
".. ... ... \n",
"275 210 EUR Sonderbonus 42.00840 \n",
"276 50.41176 \n",
"277 210 EUR Sonderbonus 50.41176 \n",
"278 46.21008 \n",
"279 210 EUR Sonderbonus 46.21008 \n",
"\n",
" starts_deal stops_deal created_deal updated_deal \n",
"0 2025-04-22 11:43:59 None 2025-04-22 11:43:59 2025-04-22 11:43:59 \n",
"1 2025-04-22 11:43:59 None 2025-04-22 11:43:59 2025-04-22 11:43:59 \n",
"2 2025-04-22 11:43:59 None 2025-04-22 11:43:59 2025-04-22 11:43:59 \n",
"3 2025-04-22 11:44:22 None 2025-04-22 11:44:22 2025-04-22 11:44:22 \n",
"4 2025-04-22 11:44:22 None 2025-04-22 11:44:22 2025-04-22 11:44:22 \n",
".. ... ... ... ... \n",
"275 2025-04-22 12:17:55 None 2025-04-22 12:17:55 2025-04-22 12:17:55 \n",
"276 2025-04-22 12:18:13 None 2025-04-22 12:18:13 2025-04-22 12:18:13 \n",
"277 2025-04-22 12:18:13 None 2025-04-22 12:18:13 2025-04-22 12:18:13 \n",
"278 2025-04-22 12:18:31 None 2025-04-22 12:18:31 2025-04-22 12:18:31 \n",
"279 2025-04-22 12:18:31 None 2025-04-22 12:18:31 2025-04-22 12:18:31 \n",
"\n",
"[280 rows x 9 columns]"
] ]
}, },
"metadata": {}, "metadata": {},
...@@ -101,17 +435,208 @@ ...@@ -101,17 +435,208 @@
" <thead>\n", " <thead>\n",
" <tr style=\"text-align: right;\">\n", " <tr style=\"text-align: right;\">\n",
" <th></th>\n", " <th></th>\n",
" <th>id_opti</th>\n",
" <th>base_opti</th>\n",
" <th>providercode_opti</th>\n",
" <th>providercategory_opti</th>\n",
" <th>name_opti</th>\n",
" <th>price_opti</th>\n",
" <th>starts_opti</th>\n",
" <th>stops_opti</th>\n",
" <th>created_opti</th>\n",
" <th>updated_opti</th>\n",
" </tr>\n", " </tr>\n",
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4627</td>\n",
" <td>133</td>\n",
" <td>O3132980</td>\n",
" <td>G3120771</td>\n",
" <td>Aktionsguthaben VP | 192 EUR Aktionsguthaben V...</td>\n",
" <td>0.00000</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>4628</td>\n",
" <td>133</td>\n",
" <td>O1173</td>\n",
" <td>G19</td>\n",
" <td>Standard EVN | EVN gekürzt</td>\n",
" <td>0.00000</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>4629</td>\n",
" <td>133</td>\n",
" <td>O15695</td>\n",
" <td>G209</td>\n",
" <td>Sperren | Sperre anonyme Anrufe - auf Kundenwu...</td>\n",
" <td>0.00000</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4630</td>\n",
" <td>133</td>\n",
" <td>O18978</td>\n",
" <td>G218</td>\n",
" <td>Mobile Sicherheit | freenet Handy Helfer 24 Mo...</td>\n",
" <td>4.19328</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4631</td>\n",
" <td>133</td>\n",
" <td>O17691</td>\n",
" <td>G236</td>\n",
" <td>freenet Cloud (Cloud) | Digitales Service Pake...</td>\n",
" <td>2.51261</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" <td>2025-04-22 11:43:59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5680</th>\n",
" <td>10307</td>\n",
" <td>264</td>\n",
" <td>O3501308</td>\n",
" <td>G3559799</td>\n",
" <td>Cashback Fachhandel | VP: 100 Euro Cashback</td>\n",
" <td>0.00000</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5681</th>\n",
" <td>10308</td>\n",
" <td>264</td>\n",
" <td>O3708580</td>\n",
" <td>G3711730</td>\n",
" <td>Rabatt auf Monatsgrundpreis | VP: 24 x 7 EUR R...</td>\n",
" <td>0.00000</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5682</th>\n",
" <td>10309</td>\n",
" <td>264</td>\n",
" <td>G19</td>\n",
" <td>G4</td>\n",
" <td>Einzelverbindungsnachweis | Standard EVN</td>\n",
" <td>0.00000</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5683</th>\n",
" <td>10310</td>\n",
" <td>264</td>\n",
" <td>O2805</td>\n",
" <td>G131</td>\n",
" <td>Multicard | Multicard 3</td>\n",
" <td>4.19328</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5684</th>\n",
" <td>10311</td>\n",
" <td>264</td>\n",
" <td>O202</td>\n",
" <td>G289</td>\n",
" <td>Deutschlandweite Festnetznummer kostenlos | St...</td>\n",
" <td>0.00000</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>None</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" <td>2025-04-22 12:18:31</td>\n",
" </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"<p>5685 rows × 10 columns</p>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
"Empty DataFrame\n", " id_opti base_opti providercode_opti providercategory_opti \\\n",
"Columns: []\n", "0 4627 133 O3132980 G3120771 \n",
"Index: []" "1 4628 133 O1173 G19 \n",
"2 4629 133 O15695 G209 \n",
"3 4630 133 O18978 G218 \n",
"4 4631 133 O17691 G236 \n",
"... ... ... ... ... \n",
"5680 10307 264 O3501308 G3559799 \n",
"5681 10308 264 O3708580 G3711730 \n",
"5682 10309 264 G19 G4 \n",
"5683 10310 264 O2805 G131 \n",
"5684 10311 264 O202 G289 \n",
"\n",
" name_opti price_opti \\\n",
"0 Aktionsguthaben VP | 192 EUR Aktionsguthaben V... 0.00000 \n",
"1 Standard EVN | EVN gekürzt 0.00000 \n",
"2 Sperren | Sperre anonyme Anrufe - auf Kundenwu... 0.00000 \n",
"3 Mobile Sicherheit | freenet Handy Helfer 24 Mo... 4.19328 \n",
"4 freenet Cloud (Cloud) | Digitales Service Pake... 2.51261 \n",
"... ... ... \n",
"5680 Cashback Fachhandel | VP: 100 Euro Cashback 0.00000 \n",
"5681 Rabatt auf Monatsgrundpreis | VP: 24 x 7 EUR R... 0.00000 \n",
"5682 Einzelverbindungsnachweis | Standard EVN 0.00000 \n",
"5683 Multicard | Multicard 3 4.19328 \n",
"5684 Deutschlandweite Festnetznummer kostenlos | St... 0.00000 \n",
"\n",
" starts_opti stops_opti created_opti updated_opti \n",
"0 2025-04-22 11:43:59 None 2025-04-22 11:43:59 2025-04-22 11:43:59 \n",
"1 2025-04-22 11:43:59 None 2025-04-22 11:43:59 2025-04-22 11:43:59 \n",
"2 2025-04-22 11:43:59 None 2025-04-22 11:43:59 2025-04-22 11:43:59 \n",
"3 2025-04-22 11:43:59 None 2025-04-22 11:43:59 2025-04-22 11:43:59 \n",
"4 2025-04-22 11:43:59 None 2025-04-22 11:43:59 2025-04-22 11:43:59 \n",
"... ... ... ... ... \n",
"5680 2025-04-22 12:18:31 None 2025-04-22 12:18:31 2025-04-22 12:18:31 \n",
"5681 2025-04-22 12:18:31 None 2025-04-22 12:18:31 2025-04-22 12:18:31 \n",
"5682 2025-04-22 12:18:31 None 2025-04-22 12:18:31 2025-04-22 12:18:31 \n",
"5683 2025-04-22 12:18:31 None 2025-04-22 12:18:31 2025-04-22 12:18:31 \n",
"5684 2025-04-22 12:18:31 None 2025-04-22 12:18:31 2025-04-22 12:18:31 \n",
"\n",
"[5685 rows x 10 columns]"
] ]
}, },
"metadata": {}, "metadata": {},
......
notebooks/UploadPdfsToAwsS3.ipynb
View file @ 38ae6ecf
...@@ -4,7 +4,9 @@ ...@@ -4,7 +4,9 @@
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 1,
"id": "b627c9fc-c8e6-4a16-b445-de08d76222bb", "id": "b627c9fc-c8e6-4a16-b445-de08d76222bb",
"metadata": {}, "metadata": {
"tags": []
},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
...@@ -12,7 +14,7 @@ ...@@ -12,7 +14,7 @@
"text": [ "text": [
"\n", "\n",
"--- Verarbeitung ID: 3243715 ---\n", "--- Verarbeitung ID: 3243715 ---\n",
"INFO: flyerurl_base gesetzt: https://freenetflyer.s3.amazonaws.com/flyers/3243715_flyer.pdf\n", "INFO: flyerurl_base bereits vorhanden: https://freenetflyer.s3.amazonaws.com/flyers/3243715_flyer.pdf\n",
"INFO: piburl_base gesetzt: https://freenetflyer.s3.amazonaws.com/pibs/3243715_pib.pdf\n", "INFO: piburl_base gesetzt: https://freenetflyer.s3.amazonaws.com/pibs/3243715_pib.pdf\n",
"\n", "\n",
"--- Verarbeitung ID: 3332926 ---\n", "--- Verarbeitung ID: 3332926 ---\n",
...@@ -626,7 +628,7 @@ ...@@ -626,7 +628,7 @@
" flyerKey = f\"flyers/{currentId}_flyer.pdf\"\n", " flyerKey = f\"flyers/{currentId}_flyer.pdf\"\n",
"\n", "\n",
" # Hier wird der Upload mithilfe des S3-Managers durchgeführt und die URL ermittelt.\n", " # Hier wird der Upload mithilfe des S3-Managers durchgeführt und die URL ermittelt.\n",
" flyerUrl = s3Manager.upload_file(flyerFile, flyerKey)\n", " flyerUrl = s3Manager.uploadFile(flyerFile, flyerKey)\n",
"\n", "\n",
" # In dieser Abzweigung wird geprüft, ob der Upload erfolgreich war und eine URL vorliegt.\n", " # In dieser Abzweigung wird geprüft, ob der Upload erfolgreich war und eine URL vorliegt.\n",
" if flyerUrl:\n", " if flyerUrl:\n",
...@@ -671,7 +673,7 @@ ...@@ -671,7 +673,7 @@
" pibKey = f\"pibs/{currentId}_pib.pdf\"\n", " pibKey = f\"pibs/{currentId}_pib.pdf\"\n",
"\n", "\n",
" # Hier wird der Upload mithilfe des S3-Managers durchgeführt und die URL ermittelt.\n", " # Hier wird der Upload mithilfe des S3-Managers durchgeführt und die URL ermittelt.\n",
" pibUrl = s3Manager.upload_file(pibFile, pibKey)\n", " pibUrl = s3Manager.uploadFile(pibFile, pibKey)\n",
"\n", "\n",
" # In dieser Abzweigung wird geprüft, ob der Upload erfolgreich war und eine URL vorliegt.\n", " # In dieser Abzweigung wird geprüft, ob der Upload erfolgreich war und eine URL vorliegt.\n",
" if pibUrl:\n", " if pibUrl:\n",
......
readme.md
View file @ 38ae6ecf
...@@ -24,6 +24,11 @@ docker run -it -d --restart always -p 2000:80 181802255479.dkr.ecr.eu-central-1. ...@@ -24,6 +24,11 @@ docker run -it -d --restart always -p 2000:80 181802255479.dkr.ecr.eu-central-1.
#### ####
# MAUI Data Toolkit # MAUI Data Toolkit
## Jupyter Lab ## Jupyter Lab
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment