bitbucket2lso.py hinzugefügt
This commit is contained in:
143
bitbucket2lso.py
Normal file
143
bitbucket2lso.py
Normal file
@@ -0,0 +1,143 @@
|
||||
#
|
||||
# Lädt die Berater-Daten von bitbucket runter und bei GPT wieder rauf
|
||||
# Benötigt als Argument die Sprache en oder de
|
||||
#
|
||||
# bitbucket2lso.py de
|
||||
# bitbucket2lso.py en
|
||||
#
|
||||
|
||||
import requests
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
import re
|
||||
import tempfile
|
||||
from shutil import rmtree
|
||||
from azure.storage.blob import BlobClient #pip3 install azure-storage-blob
|
||||
from requests.auth import HTTPBasicAuth
|
||||
|
||||
sprachen = {'en': {"pfad": "gmpca", "auth" : "<bitbucket token>"},
|
||||
'de': {"pfad": "gmpb", "auth" : "<bitbucket token>"}
|
||||
}
|
||||
|
||||
#tmpfolder = "/var/tmp/" # Für Linux
|
||||
tmpfolder = tempfile.gettempdir()+"/"
|
||||
|
||||
GPT_USERNAME = "<gpt user>"
|
||||
GPT_PASSWORD = "<gpt pass>"
|
||||
GPT_BASE_URL = "https://connector-gmp.myg.pt/api/v1"
|
||||
|
||||
def bitBucketRequest(path, isFile=False, page=None):
|
||||
url = "https://api.bitbucket.org/2.0/repositories/michael_lammel/"+sprachen[sprache]["pfad"]+"/src/master/"+path
|
||||
if page is not None:
|
||||
url += "?page="+page
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"Authorization": "Bearer "+sprachen[sprache]["auth"]
|
||||
}
|
||||
response = requests.request(
|
||||
"GET",
|
||||
url,
|
||||
headers=headers
|
||||
)
|
||||
print(response.text)
|
||||
if isFile:
|
||||
response.encoding = response.apparent_encoding # <-- sonst hatte ich bei manchen Dateien ein seltsames Zeichen
|
||||
return response.text
|
||||
else:
|
||||
try:
|
||||
return json.loads(response.text)
|
||||
except ValueError as e:
|
||||
print("Fehler bei "+path+": "+response.text)
|
||||
sys.exit()
|
||||
|
||||
def getFile(path):
|
||||
head_tail = os.path.split(path) # => [pfad, dateiname] in Bitbucket
|
||||
dlpath = tmpfolder + "berater-xml/" + head_tail[0]
|
||||
if os.path.isdir(dlpath) == False:
|
||||
os.makedirs(dlpath)
|
||||
result = bitBucketRequest(path, True)
|
||||
f = open(dlpath + "/" + head_tail[1], "w")
|
||||
f.write(result)
|
||||
f.close()
|
||||
|
||||
def getDirContentRecursive(dir, page=None):
|
||||
result = bitBucketRequest(dir, False, page)
|
||||
for el in result['values']:
|
||||
if el['type'] == "commit_directory" and el['path'] != "Information" and el['path'] != "Allgemeines": # Ordner 'Allgemeines'/'Information' ausschließen
|
||||
getDirContentRecursive(el['path'])
|
||||
if el['type'] == "commit_file" and el['mimetype'] == "application/xml":
|
||||
getFile(el['path'])
|
||||
# Gibt es eine weitere Seite mit mehr Ergebnissen?
|
||||
if 'next' in result:
|
||||
#m = re.search('\?page=(.*)&?', result['next'])
|
||||
m = re.search(re.escape('?')+'page=(.*)&?', result['next'])
|
||||
if (m is not None):
|
||||
getDirContentRecursive(dir, m.group(1))
|
||||
|
||||
def zipdir(ziph):
|
||||
for root, dirs, files in os.walk(tmpfolder + "berater-xml/"):
|
||||
for file in files:
|
||||
ziph.write(os.path.join(root, file),
|
||||
os.path.relpath(os.path.join(root, file),
|
||||
os.path.join(tmpfolder + "berater-xml/", '..')))
|
||||
|
||||
def upload():
|
||||
# first acquire the azure blob storage upload url
|
||||
auth = HTTPBasicAuth(username=GPT_USERNAME, password=GPT_PASSWORD)
|
||||
upload_url_resp = requests.get(f"{GPT_BASE_URL}/upload-url", auth=auth)
|
||||
upload_url_json = json.loads(upload_url_resp.content.decode("utf-8"))
|
||||
upload_url = upload_url_json["url"]
|
||||
|
||||
# once the url has been acquired the file can be uploaded using azure blob client
|
||||
blob_client = BlobClient.from_blob_url(upload_url)
|
||||
with open(tmpfolder+'berater-'+sprache+'.zip', mode="rb") as f:
|
||||
blob_client.upload_blob(data=f.read())
|
||||
|
||||
print("Datei hochgeladen. Stoße nun Processing-Queue für Verarbeitung an.")
|
||||
|
||||
# after the successful upload the original body from the first request
|
||||
# must be sent to initiate the processing pipeline for a given language (selected by using the correct path parameter)
|
||||
# the response is a text/event-stream containing the progress of the processing pipeline
|
||||
# note: the processing pipeline is asynchronous and the response does not indicate the final state of the pipeline
|
||||
session = requests.Session()
|
||||
#requests.post(f"{GPT_BASE_URL}/"+sprache+"/process", json=upload_url_json, auth=auth) # Alte Version ohne Log-Stream
|
||||
with session.post(f"{GPT_BASE_URL}/"+sprache+"/process", json=upload_url_json, auth=auth, stream=True) as response:
|
||||
for line in response.iter_lines():
|
||||
if line:
|
||||
print(line)
|
||||
|
||||
# Start
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if len(sys.argv) < 2 or (sys.argv[1] != 'de' and sys.argv[1] != 'en'):
|
||||
print("Bitte als Argument die Sprache de (für BERATER) oder en (für Adviser) mitgeben.")
|
||||
sys.exit()
|
||||
|
||||
sprache = sys.argv[1]
|
||||
|
||||
if os.path.isdir(tmpfolder + "berater-xml/") == False:
|
||||
os.mkdir(tmpfolder + "berater-xml/")
|
||||
|
||||
# Ordner rekursiv durchgehen und xml-Dateien herunterladen
|
||||
print("Lade Dateien aus bitbucket herunter...das kann eine Weile dauern...")
|
||||
result = getDirContentRecursive("")
|
||||
|
||||
# Heruntergeladene Dateien zippen
|
||||
print("Generiere ZIP-Datei")
|
||||
with zipfile.ZipFile(tmpfolder+'berater-'+sprache+'.zip', 'w', zipfile.ZIP_DEFLATED) as zipf:
|
||||
zipdir(zipf)
|
||||
|
||||
# Dowloadordner wieder löschen
|
||||
rmtree(tmpfolder + "berater-xml/")
|
||||
|
||||
# Nun bei GPT hochladen
|
||||
print("Daten werden hochgeladen. Bitte folgendes Log beachten:")
|
||||
#upload()
|
||||
|
||||
print(" ")
|
||||
print("Transfer abgeschlossen.")
|
||||
print("Sofern das Log obendrüber keine Fehler anzeigt, sollte der weitere Fortschritt des Importvorganges in der KnowledgeBase zu sehen sein: https://gpt.gmp-verlag.de/admin/knowledge-bases")
|
||||
#os.remove(tmpfolder+'berater-'+sprache+'.zip')
|
||||
Reference in New Issue
Block a user