From 7ab50266026aadb935fc2bf97af19f6cf1c1bb5b Mon Sep 17 00:00:00 2001 From: Mirco Ropic Date: Thu, 16 Jul 2026 12:19:44 +0000 Subject: [PATCH] =?UTF-8?q?bitbucket2lso.py=20hinzugef=C3=BCgt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bitbucket2lso.py | 143 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 bitbucket2lso.py diff --git a/bitbucket2lso.py b/bitbucket2lso.py new file mode 100644 index 0000000..853adb2 --- /dev/null +++ b/bitbucket2lso.py @@ -0,0 +1,143 @@ +# +# Lädt die Berater-Daten von bitbucket runter und bei GPT wieder rauf +# Benötigt als Argument die Sprache en oder de +# +# bitbucket2lso.py de +# bitbucket2lso.py en +# + +import requests +import json +import os +import sys +import zipfile +import re +import tempfile +from shutil import rmtree +from azure.storage.blob import BlobClient #pip3 install azure-storage-blob +from requests.auth import HTTPBasicAuth + +sprachen = {'en': {"pfad": "gmpca", "auth" : ""}, + 'de': {"pfad": "gmpb", "auth" : ""} + } + +#tmpfolder = "/var/tmp/" # Für Linux +tmpfolder = tempfile.gettempdir()+"/" + +GPT_USERNAME = "" +GPT_PASSWORD = "" +GPT_BASE_URL = "https://connector-gmp.myg.pt/api/v1" + +def bitBucketRequest(path, isFile=False, page=None): + url = "https://api.bitbucket.org/2.0/repositories/michael_lammel/"+sprachen[sprache]["pfad"]+"/src/master/"+path + if page is not None: + url += "?page="+page + headers = { + "Accept": "application/json", + "Authorization": "Bearer "+sprachen[sprache]["auth"] + } + response = requests.request( + "GET", + url, + headers=headers + ) + print(response.text) + if isFile: + response.encoding = response.apparent_encoding # <-- sonst hatte ich bei manchen Dateien ein seltsames Zeichen + return response.text + else: + try: + return json.loads(response.text) + except ValueError as e: + print("Fehler bei "+path+": "+response.text) + sys.exit() + +def getFile(path): + head_tail = os.path.split(path) # => [pfad, dateiname] in Bitbucket + dlpath = tmpfolder + "berater-xml/" + head_tail[0] + if os.path.isdir(dlpath) == False: + os.makedirs(dlpath) + result = bitBucketRequest(path, True) + f = open(dlpath + "/" + head_tail[1], "w") + f.write(result) + f.close() + +def getDirContentRecursive(dir, page=None): + result = bitBucketRequest(dir, False, page) + for el in result['values']: + if el['type'] == "commit_directory" and el['path'] != "Information" and el['path'] != "Allgemeines": # Ordner 'Allgemeines'/'Information' ausschließen + getDirContentRecursive(el['path']) + if el['type'] == "commit_file" and el['mimetype'] == "application/xml": + getFile(el['path']) + # Gibt es eine weitere Seite mit mehr Ergebnissen? + if 'next' in result: + #m = re.search('\?page=(.*)&?', result['next']) + m = re.search(re.escape('?')+'page=(.*)&?', result['next']) + if (m is not None): + getDirContentRecursive(dir, m.group(1)) + +def zipdir(ziph): + for root, dirs, files in os.walk(tmpfolder + "berater-xml/"): + for file in files: + ziph.write(os.path.join(root, file), + os.path.relpath(os.path.join(root, file), + os.path.join(tmpfolder + "berater-xml/", '..'))) + +def upload(): + # first acquire the azure blob storage upload url + auth = HTTPBasicAuth(username=GPT_USERNAME, password=GPT_PASSWORD) + upload_url_resp = requests.get(f"{GPT_BASE_URL}/upload-url", auth=auth) + upload_url_json = json.loads(upload_url_resp.content.decode("utf-8")) + upload_url = upload_url_json["url"] + + # once the url has been acquired the file can be uploaded using azure blob client + blob_client = BlobClient.from_blob_url(upload_url) + with open(tmpfolder+'berater-'+sprache+'.zip', mode="rb") as f: + blob_client.upload_blob(data=f.read()) + + print("Datei hochgeladen. Stoße nun Processing-Queue für Verarbeitung an.") + + # after the successful upload the original body from the first request + # must be sent to initiate the processing pipeline for a given language (selected by using the correct path parameter) + # the response is a text/event-stream containing the progress of the processing pipeline + # note: the processing pipeline is asynchronous and the response does not indicate the final state of the pipeline + session = requests.Session() + #requests.post(f"{GPT_BASE_URL}/"+sprache+"/process", json=upload_url_json, auth=auth) # Alte Version ohne Log-Stream + with session.post(f"{GPT_BASE_URL}/"+sprache+"/process", json=upload_url_json, auth=auth, stream=True) as response: + for line in response.iter_lines(): + if line: + print(line) + +# Start + +if __name__ == '__main__': + + if len(sys.argv) < 2 or (sys.argv[1] != 'de' and sys.argv[1] != 'en'): + print("Bitte als Argument die Sprache de (für BERATER) oder en (für Adviser) mitgeben.") + sys.exit() + + sprache = sys.argv[1] + + if os.path.isdir(tmpfolder + "berater-xml/") == False: + os.mkdir(tmpfolder + "berater-xml/") + + # Ordner rekursiv durchgehen und xml-Dateien herunterladen + print("Lade Dateien aus bitbucket herunter...das kann eine Weile dauern...") + result = getDirContentRecursive("") + + # Heruntergeladene Dateien zippen + print("Generiere ZIP-Datei") + with zipfile.ZipFile(tmpfolder+'berater-'+sprache+'.zip', 'w', zipfile.ZIP_DEFLATED) as zipf: + zipdir(zipf) + + # Dowloadordner wieder löschen + rmtree(tmpfolder + "berater-xml/") + + # Nun bei GPT hochladen + print("Daten werden hochgeladen. Bitte folgendes Log beachten:") + #upload() + + print(" ") + print("Transfer abgeschlossen.") + print("Sofern das Log obendrüber keine Fehler anzeigt, sollte der weitere Fortschritt des Importvorganges in der KnowledgeBase zu sehen sein: https://gpt.gmp-verlag.de/admin/knowledge-bases") + #os.remove(tmpfolder+'berater-'+sprache+'.zip') \ No newline at end of file