bitbucket2lso.py hinzugefügt
This commit is contained in:
143
bitbucket2lso.py
Normal file
143
bitbucket2lso.py
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
#
|
||||||
|
# Lädt die Berater-Daten von bitbucket runter und bei GPT wieder rauf
|
||||||
|
# Benötigt als Argument die Sprache en oder de
|
||||||
|
#
|
||||||
|
# bitbucket2lso.py de
|
||||||
|
# bitbucket2lso.py en
|
||||||
|
#
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
from shutil import rmtree
|
||||||
|
from azure.storage.blob import BlobClient #pip3 install azure-storage-blob
|
||||||
|
from requests.auth import HTTPBasicAuth
|
||||||
|
|
||||||
|
sprachen = {'en': {"pfad": "gmpca", "auth" : "<bitbucket token>"},
|
||||||
|
'de': {"pfad": "gmpb", "auth" : "<bitbucket token>"}
|
||||||
|
}
|
||||||
|
|
||||||
|
#tmpfolder = "/var/tmp/" # Für Linux
|
||||||
|
tmpfolder = tempfile.gettempdir()+"/"
|
||||||
|
|
||||||
|
GPT_USERNAME = "<gpt user>"
|
||||||
|
GPT_PASSWORD = "<gpt pass>"
|
||||||
|
GPT_BASE_URL = "https://connector-gmp.myg.pt/api/v1"
|
||||||
|
|
||||||
|
def bitBucketRequest(path, isFile=False, page=None):
|
||||||
|
url = "https://api.bitbucket.org/2.0/repositories/michael_lammel/"+sprachen[sprache]["pfad"]+"/src/master/"+path
|
||||||
|
if page is not None:
|
||||||
|
url += "?page="+page
|
||||||
|
headers = {
|
||||||
|
"Accept": "application/json",
|
||||||
|
"Authorization": "Bearer "+sprachen[sprache]["auth"]
|
||||||
|
}
|
||||||
|
response = requests.request(
|
||||||
|
"GET",
|
||||||
|
url,
|
||||||
|
headers=headers
|
||||||
|
)
|
||||||
|
print(response.text)
|
||||||
|
if isFile:
|
||||||
|
response.encoding = response.apparent_encoding # <-- sonst hatte ich bei manchen Dateien ein seltsames Zeichen
|
||||||
|
return response.text
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
return json.loads(response.text)
|
||||||
|
except ValueError as e:
|
||||||
|
print("Fehler bei "+path+": "+response.text)
|
||||||
|
sys.exit()
|
||||||
|
|
||||||
|
def getFile(path):
|
||||||
|
head_tail = os.path.split(path) # => [pfad, dateiname] in Bitbucket
|
||||||
|
dlpath = tmpfolder + "berater-xml/" + head_tail[0]
|
||||||
|
if os.path.isdir(dlpath) == False:
|
||||||
|
os.makedirs(dlpath)
|
||||||
|
result = bitBucketRequest(path, True)
|
||||||
|
f = open(dlpath + "/" + head_tail[1], "w")
|
||||||
|
f.write(result)
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
def getDirContentRecursive(dir, page=None):
|
||||||
|
result = bitBucketRequest(dir, False, page)
|
||||||
|
for el in result['values']:
|
||||||
|
if el['type'] == "commit_directory" and el['path'] != "Information" and el['path'] != "Allgemeines": # Ordner 'Allgemeines'/'Information' ausschließen
|
||||||
|
getDirContentRecursive(el['path'])
|
||||||
|
if el['type'] == "commit_file" and el['mimetype'] == "application/xml":
|
||||||
|
getFile(el['path'])
|
||||||
|
# Gibt es eine weitere Seite mit mehr Ergebnissen?
|
||||||
|
if 'next' in result:
|
||||||
|
#m = re.search('\?page=(.*)&?', result['next'])
|
||||||
|
m = re.search(re.escape('?')+'page=(.*)&?', result['next'])
|
||||||
|
if (m is not None):
|
||||||
|
getDirContentRecursive(dir, m.group(1))
|
||||||
|
|
||||||
|
def zipdir(ziph):
|
||||||
|
for root, dirs, files in os.walk(tmpfolder + "berater-xml/"):
|
||||||
|
for file in files:
|
||||||
|
ziph.write(os.path.join(root, file),
|
||||||
|
os.path.relpath(os.path.join(root, file),
|
||||||
|
os.path.join(tmpfolder + "berater-xml/", '..')))
|
||||||
|
|
||||||
|
def upload():
|
||||||
|
# first acquire the azure blob storage upload url
|
||||||
|
auth = HTTPBasicAuth(username=GPT_USERNAME, password=GPT_PASSWORD)
|
||||||
|
upload_url_resp = requests.get(f"{GPT_BASE_URL}/upload-url", auth=auth)
|
||||||
|
upload_url_json = json.loads(upload_url_resp.content.decode("utf-8"))
|
||||||
|
upload_url = upload_url_json["url"]
|
||||||
|
|
||||||
|
# once the url has been acquired the file can be uploaded using azure blob client
|
||||||
|
blob_client = BlobClient.from_blob_url(upload_url)
|
||||||
|
with open(tmpfolder+'berater-'+sprache+'.zip', mode="rb") as f:
|
||||||
|
blob_client.upload_blob(data=f.read())
|
||||||
|
|
||||||
|
print("Datei hochgeladen. Stoße nun Processing-Queue für Verarbeitung an.")
|
||||||
|
|
||||||
|
# after the successful upload the original body from the first request
|
||||||
|
# must be sent to initiate the processing pipeline for a given language (selected by using the correct path parameter)
|
||||||
|
# the response is a text/event-stream containing the progress of the processing pipeline
|
||||||
|
# note: the processing pipeline is asynchronous and the response does not indicate the final state of the pipeline
|
||||||
|
session = requests.Session()
|
||||||
|
#requests.post(f"{GPT_BASE_URL}/"+sprache+"/process", json=upload_url_json, auth=auth) # Alte Version ohne Log-Stream
|
||||||
|
with session.post(f"{GPT_BASE_URL}/"+sprache+"/process", json=upload_url_json, auth=auth, stream=True) as response:
|
||||||
|
for line in response.iter_lines():
|
||||||
|
if line:
|
||||||
|
print(line)
|
||||||
|
|
||||||
|
# Start
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
if len(sys.argv) < 2 or (sys.argv[1] != 'de' and sys.argv[1] != 'en'):
|
||||||
|
print("Bitte als Argument die Sprache de (für BERATER) oder en (für Adviser) mitgeben.")
|
||||||
|
sys.exit()
|
||||||
|
|
||||||
|
sprache = sys.argv[1]
|
||||||
|
|
||||||
|
if os.path.isdir(tmpfolder + "berater-xml/") == False:
|
||||||
|
os.mkdir(tmpfolder + "berater-xml/")
|
||||||
|
|
||||||
|
# Ordner rekursiv durchgehen und xml-Dateien herunterladen
|
||||||
|
print("Lade Dateien aus bitbucket herunter...das kann eine Weile dauern...")
|
||||||
|
result = getDirContentRecursive("")
|
||||||
|
|
||||||
|
# Heruntergeladene Dateien zippen
|
||||||
|
print("Generiere ZIP-Datei")
|
||||||
|
with zipfile.ZipFile(tmpfolder+'berater-'+sprache+'.zip', 'w', zipfile.ZIP_DEFLATED) as zipf:
|
||||||
|
zipdir(zipf)
|
||||||
|
|
||||||
|
# Dowloadordner wieder löschen
|
||||||
|
rmtree(tmpfolder + "berater-xml/")
|
||||||
|
|
||||||
|
# Nun bei GPT hochladen
|
||||||
|
print("Daten werden hochgeladen. Bitte folgendes Log beachten:")
|
||||||
|
#upload()
|
||||||
|
|
||||||
|
print(" ")
|
||||||
|
print("Transfer abgeschlossen.")
|
||||||
|
print("Sofern das Log obendrüber keine Fehler anzeigt, sollte der weitere Fortschritt des Importvorganges in der KnowledgeBase zu sehen sein: https://gpt.gmp-verlag.de/admin/knowledge-bases")
|
||||||
|
#os.remove(tmpfolder+'berater-'+sprache+'.zip')
|
||||||
Reference in New Issue
Block a user