144 lines
5.1 KiB
Python
144 lines
5.1 KiB
Python
#
|
|
# Lädt die Berater-Daten von bitbucket runter und bei GPT wieder rauf
|
|
# Benötigt als Argument die Sprache en oder de
|
|
#
|
|
# bitbucket2lso.py de
|
|
# bitbucket2lso.py en
|
|
#
|
|
|
|
import requests
|
|
import json
|
|
import os
|
|
import sys
|
|
import zipfile
|
|
import re
|
|
import tempfile
|
|
from shutil import rmtree
|
|
from azure.storage.blob import BlobClient #pip3 install azure-storage-blob
|
|
from requests.auth import HTTPBasicAuth
|
|
|
|
sprachen = {'en': {"pfad": "gmpca"},
|
|
'de': {"pfad": "gmpb"}
|
|
}
|
|
|
|
#tmpfolder = "/var/tmp/" # Für Linux
|
|
tmpfolder = tempfile.gettempdir()+"/"
|
|
|
|
BITBUCKET_AUTH = "<bitbucket token mit email bash64 hashed>"
|
|
|
|
GPT_USERNAME = "<gpt user>"
|
|
GPT_PASSWORD = "<gpt pass>"
|
|
GPT_BASE_URL = "https://connector-gmp.myg.pt/api/v1"
|
|
|
|
def bitBucketRequest(path, isFile=False, page=None):
|
|
url = "https://api.bitbucket.org/2.0/repositories/michael_lammel/"+sprachen[sprache]["pfad"]+"/src/master/"+path
|
|
if page is not None:
|
|
url += "?page="+page
|
|
headers = {
|
|
"Accept": "application/json",
|
|
"Authorization": "Basic "+BITBUCKET_AUTH
|
|
}
|
|
response = requests.request(
|
|
"GET",
|
|
url,
|
|
headers=headers
|
|
)
|
|
if isFile:
|
|
response.encoding = response.apparent_encoding # <-- sonst hatte ich bei manchen Dateien ein seltsames Zeichen
|
|
return response.text
|
|
else:
|
|
try:
|
|
return json.loads(response.text)
|
|
except ValueError as e:
|
|
print("Fehler bei "+path+": "+response.text)
|
|
sys.exit()
|
|
|
|
def getFile(path):
|
|
head_tail = os.path.split(path) # => [pfad, dateiname] in Bitbucket
|
|
dlpath = tmpfolder + "berater-xml/" + head_tail[0]
|
|
if os.path.isdir(dlpath) == False:
|
|
os.makedirs(dlpath)
|
|
result = bitBucketRequest(path, True)
|
|
f = open(dlpath + "/" + head_tail[1], "w")
|
|
f.write(result)
|
|
f.close()
|
|
|
|
def getDirContentRecursive(dir, page=None):
|
|
result = bitBucketRequest(dir, False, page)
|
|
for el in result['values']:
|
|
if el['type'] == "commit_directory" and el['path'] != "Information" and el['path'] != "Allgemeines": # Ordner 'Allgemeines'/'Information' ausschließen
|
|
getDirContentRecursive(el['path'])
|
|
if el['type'] == "commit_file" and el['mimetype'] == "application/xml":
|
|
getFile(el['path'])
|
|
# Gibt es eine weitere Seite mit mehr Ergebnissen?
|
|
if 'next' in result:
|
|
#m = re.search('\?page=(.*)&?', result['next'])
|
|
m = re.search(re.escape('?')+'page=(.*)&?', result['next'])
|
|
if (m is not None):
|
|
getDirContentRecursive(dir, m.group(1))
|
|
|
|
def zipdir(ziph):
|
|
for root, dirs, files in os.walk(tmpfolder + "berater-xml/"):
|
|
for file in files:
|
|
ziph.write(os.path.join(root, file),
|
|
os.path.relpath(os.path.join(root, file),
|
|
os.path.join(tmpfolder + "berater-xml/", '..')))
|
|
|
|
def upload():
|
|
# first acquire the azure blob storage upload url
|
|
auth = HTTPBasicAuth(username=GPT_USERNAME, password=GPT_PASSWORD)
|
|
upload_url_resp = requests.get(f"{GPT_BASE_URL}/upload-url", auth=auth)
|
|
upload_url_json = json.loads(upload_url_resp.content.decode("utf-8"))
|
|
upload_url = upload_url_json["url"]
|
|
|
|
# once the url has been acquired the file can be uploaded using azure blob client
|
|
blob_client = BlobClient.from_blob_url(upload_url)
|
|
with open(tmpfolder+'berater-'+sprache+'.zip', mode="rb") as f:
|
|
blob_client.upload_blob(data=f.read())
|
|
|
|
print("Datei hochgeladen. Stoße nun Processing-Queue für Verarbeitung an.")
|
|
|
|
# after the successful upload the original body from the first request
|
|
# must be sent to initiate the processing pipeline for a given language (selected by using the correct path parameter)
|
|
# the response is a text/event-stream containing the progress of the processing pipeline
|
|
# note: the processing pipeline is asynchronous and the response does not indicate the final state of the pipeline
|
|
session = requests.Session()
|
|
#requests.post(f"{GPT_BASE_URL}/"+sprache+"/process", json=upload_url_json, auth=auth) # Alte Version ohne Log-Stream
|
|
with session.post(f"{GPT_BASE_URL}/"+sprache+"/process", json=upload_url_json, auth=auth, stream=True) as response:
|
|
for line in response.iter_lines():
|
|
if line:
|
|
print(line)
|
|
|
|
# Start
|
|
|
|
if __name__ == '__main__':
|
|
|
|
if len(sys.argv) < 2 or (sys.argv[1] != 'de' and sys.argv[1] != 'en'):
|
|
print("Bitte als Argument die Sprache de (für BERATER) oder en (für Adviser) mitgeben.")
|
|
sys.exit()
|
|
|
|
sprache = sys.argv[1]
|
|
|
|
if os.path.isdir(tmpfolder + "berater-xml/") == False:
|
|
os.mkdir(tmpfolder + "berater-xml/")
|
|
|
|
# Ordner rekursiv durchgehen und xml-Dateien herunterladen
|
|
print("Lade Dateien aus bitbucket herunter...das kann eine Weile dauern...")
|
|
result = getDirContentRecursive("")
|
|
|
|
# Heruntergeladene Dateien zippen
|
|
print("Generiere ZIP-Datei")
|
|
with zipfile.ZipFile(tmpfolder+'berater-'+sprache+'.zip', 'w', zipfile.ZIP_DEFLATED) as zipf:
|
|
zipdir(zipf)
|
|
|
|
# Dowloadordner wieder löschen
|
|
rmtree(tmpfolder + "berater-xml/")
|
|
|
|
# Nun bei GPT hochladen
|
|
print("Daten werden hochgeladen. Bitte folgendes Log beachten:")
|
|
upload()
|
|
|
|
print(" ")
|
|
print("Transfer abgeschlossen.")
|
|
print("Sofern das Log obendrüber keine Fehler anzeigt, sollte der weitere Fortschritt des Importvorganges in der KnowledgeBase zu sehen sein: https://gpt.gmp-verlag.de/admin/knowledge-bases")
|
|
os.remove(tmpfolder+'berater-'+sprache+'.zip') |