8000 fix: remote upload error by siiddhantt · Pull Request #933 · arc53/DocsGPT · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

fix: remote upload error #933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 3, 2024
Merged

fix: remote upload error #933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aa670ef
fix: connection aborted in WebBaseLoader
siiddhantt May 3, 2024
53e8620
fix: added more headers from default
siiddhantt May 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 91 additions & 54 deletions application/api/user/routes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@
prompts_collection = db["prompts"]
feedback_collection = db["feedback"]
api_key_collection = db["api_keys"]
user = Blueprint('user', __name__)
user = Blueprint("user", __name__)

current_dir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)

current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

@user.route("/api/delete_conversation", methods=["POST"])
def delete_conversation():
Expand All @@ -37,21 +40,25 @@ def delete_conversation():

return {"status": "ok"}


@user.route("/api/delete_all_conversations", methods=["POST"])
def delete_all_conversations():
user_id = "local"
conversations_collection.delete_many({"user":user_id})
conversations_collection.delete_many({"user": user_id})
return {"status": "ok"}


@user.route("/api/get_conversations", methods=["get"])
def get_conversations():
# provides a list of conversations
conversations = conversations_collection.find().sort("date", -1).limit(30)
list_conversations = []
for conversation in conversations:
list_conversations.append({"id": str(conversation["_id"]), "name": conversation["name"]})
list_conversations.append(
{"id": str(conversation["_id"]), "name": conversation["name"]}
)

#list_conversations = [{"id": "default", "name": "default"}, {"id": "jeff", "name": "jeff"}]
# list_conversations = [{"id": "default", "name": "default"}, {"id": "jeff", "name": "jeff"}]

return jsonify(list_conversations)

Expand All @@ -61,15 +68,16 @@ def get_single_conversation():
# provides data for a conversation
conversation_id = request.args.get("id")
conversation = conversations_collection.find_one({"_id": ObjectId(conversation_id)})
return jsonify(conversation['queries'])
return jsonify(conversation["queries"])


@user.route("/api/update_conversation_name", methods=["POST"])
def update_conversation_name():
# update data for a conversation
data = request.get_json()
id = data["id"]
name = data["name"]
conversations_collection.update_one({"_id": ObjectId(id)},{"$set":{"name":name}})
conversations_collection.update_one({"_id": ObjectId(id)}, {"$set": {"name": name}})
return {"status": "ok"}


Expand All @@ -80,7 +88,6 @@ def api_feedback():
answer = data["answer"]
feedback = data["feedback"]


feedback_collection.insert_one(
{
"question": question,
Expand All @@ -90,6 +97,7 @@ def api_feedback():
)
return {"status": "ok"}


@user.route("/api/delete_by_ids", methods=["get"])
def delete_by_ids():
"""Delete by ID. These are the IDs in the vectorstore"""
Expand All @@ -104,6 +112,7 @@ def delete_by_ids():
return {"status": "ok"}
return {"status": "error"}


@user.route("/api/delete_old", methods=["get"])
def delete_old():
"""Delete old indexes."""
Expand All @@ -119,7 +128,7 @@ def delete_old():
if dirs_clean[0] not in ["indexes", "vectors"]:
return {"status": "error"}
path_clean = "/".join(dirs_clean)
vectors_collection.delete_one({"name": dirs_clean[-1], 'user': dirs_clean[-2]})
vectors_collection.delete_one({"name": dirs_clean[-1], "user": dirs_clean[-2]})
if settings.VECTOR_STORE == "faiss":
try:
shutil.rmtree(os.path.join(current_dir, path_clean))
Expand All @@ -130,9 +139,10 @@ def delete_old():
settings.VECTOR_STORE, path=os.path.join(current_dir, path_clean)
)
vetorstore.delete_index()

return {"status": "ok"}


@user.route("/api/upload", methods=["POST"])
def upload_file():
"""Upload a file to get vectorized and indexed."""
Expand All @@ -144,27 +154,29 @@ def upload_file():
job_name = secure_filename(request.form["name"])
# check if the post request has the file part
files = request.files.getlist("file")
if not files or all(file.filename == '' for file in files):

if not files or all(file.filename == "" for file in files):
return {"status": "no file name"}

# Directory where files will be saved
save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, job_name)
os.makedirs(save_dir, exist_ok=True)

if len(files) > 1:
# Multiple files; prepare them for zip
temp_dir = os.path.join(save_dir, "temp")
os.makedirs(temp_dir, exist_ok=True)

for file in files:
filename = secure_filename(file.filename)
file.save(os.path.join(temp_dir, filename))

# Use shutil.make_archive to zip the temp directory
zip_path = shutil.make_archive(base_name=os.path.join(save_dir, job_name), format='zip', root_dir=temp_dir)
zip_path = shutil.make_archive(
base_name=os.path.join(save_dir, job_name), format="zip", root_dir=temp_dir
)
final_filename = os.path.basename(zip_path)

# Clean up the temporary directory after zipping
shutil.rmtree(temp_dir)
else:
Expand All @@ -173,14 +185,19 @@ def upload_file():
final_filename = secure_filename(file.filename)
file_path = os.path.join(save_dir, final_filename)
file.save(file_path)

# Call ingest with the single file or zipped file
task = ingest.delay(settings.UPLOAD_FOLDER, [".rst", ".md", ".pdf", ".txt", ".docx",
".csv", ".epub", ".html", &q 10000 uot;.mdx"],
job_name, final_filename, user)

task = ingest.delay(
settings.UPLOAD_FOLDER,
[".rst", ".md", ".pdf", ".txt", ".docx", ".csv", ".epub", ".html", ".mdx"],
job_name,
final_filename,
user,
)

return {"status": "ok", "task_id": task.id}



@user.route("/api/remote", methods=["POST"])
def upload_remote():
"""Upload a remote source to get vectorized and indexed."""
Expand All @@ -193,25 +210,27 @@ def upload_remote():
if "name" not in request.form:
return {"status": "no name"}
job_name = secure_filename(request.form["name"])
# check if the post request has the file part
if "data" not in request.form:
print("No data")
return {"status": "no data"}
source_data = request.form["data"]

if source_data:
task = ingest_remote.delay(source_data=source_data, job_name=job_name, user=user, loader=source)
# task id
task = ingest_remote.delay(
source_data=source_data, job_name=job_name, user=user, loader=source
)
task_id = task.id
return {"status": "ok", "task_id": task_id}
else:
return {"status": "error"}


@user.route("/api/task_status", methods=["GET"])
def task_status():
"""Get celery job status."""
task_id = request.args.get("task_id")
from application.celery import celery

task = celery.AsyncResult(task_id)
task_meta = task.info
return {"status": task.status, "result": task_meta}
Expand Down Expand Up @@ -253,11 +272,13 @@ def combined_json():
}
)
if settings.VECTOR_STORE == "faiss":
data_remote = requests.get("https://d3dg1063dc54p9.cloudfront.net/combined.json").json()
data_remote = requests.get(
"https://d3dg1063dc54p9.cloudfront.net/combined.json"
).json()
for index in data_remote:
index["location"] = "remote"
data.append(index)
if 'duckduck_search' in settings.RETRIEVERS_ENABLED:
if "duckduck_search" in settings.RETRIEVERS_ENABLED:
data.append(
{
"name": "DuckDuckGo Search",
Expand All @@ -271,7 +292,7 @@ def combined_json():
"location": "custom",
}
)
if 'brave_search' in settings.RETRIEVERS_ENABLED:
if "brave_search" in settings.RETRIEVERS_ENABLED:
data.append(
{
"name": "Brave Search",
Expand Down Expand Up @@ -302,11 +323,11 @@ def check_docs():
return {"status": "exists"}
else:
file_url = urlparse(base_path + vectorstore + "index.faiss")

if (
file_url.scheme in ['https'] and
file_url.netloc == 'raw.githubusercontent.com' and
file_url.path.startswith('/arc53/DocsHUB/main/')
file_url.scheme in ["https"]
and file_url.netloc == "raw.githubusercontent.com"
and file_url.path.startswith("/arc53/DocsHUB/main/")
):
r = requests.get(file_url.geturl())
if r.status_code != 200:
Expand All @@ -325,6 +346,7 @@ def check_docs():

return {"status": "loaded"}


@user.route("/api/create_prompt", methods=["POST"])
def create_prompt():
data = request.get_json()
Expand All @@ -343,6 +365,7 @@ def create_prompt():
new_id = str(resp.inserted_id)
return {"id": new_id}


@user.route("/api/get_prompts", methods=["GET"])
def get_prompts():
user = "local"
Expand All @@ -352,30 +375,39 @@ def get_prompts():
list_prompts.append({"id": "creative", "name": "creative", "type": "public"})
list_prompts.append({"id": "strict", "name": "strict", "type": "public"})
for prompt in prompts:
list_prompts.append({"id": str(prompt["_id"]), "name": prompt["name"], "type": "private"})
list_prompts.append(
{"id": str(prompt["_id"]), "name": prompt["name"], "type": "private"}
)

return jsonify(list_prompts)


@user.route("/api/get_single_prompt", methods=["GET"])
def get_single_prompt():
prompt_id = request.args.get("id")
if prompt_id == 'default':
with open(os.path.join(current_dir, "prompts", "chat_combine_default.txt"), "r") as f:
if prompt_id == "default":
with open(
os.path.join(current_dir, "prompts", "chat_combine_default.txt"), "r"
) as f:
chat_combine_template = f.read()
return jsonify({"content": chat_combine_template})
elif prompt_id == 'creative':
with open(os.path.join(current_dir, "prompts", "chat_combine_creative.txt"), "r") as f:
elif prompt_id == "creative":
with open(
os.path.join(current_dir, "prompts", "chat_combine_creative.txt"), "r"
) as f:
chat_reduce_creative = f.read()
return jsonify({"content": chat_reduce_creative})
elif prompt_id == 'strict':
with open(os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r") as f:
chat_reduce_strict = f.read()
elif prompt_id == "strict":
with open(
os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r"
) as f:
chat_reduce_strict = f.read()
return jsonify({"content": chat_reduce_strict})


prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)})
return jsonify({"content": prompt["content"]})


@user.route("/api/delete_prompt", methods=["POST"])
def delete_prompt():
data = request.get_json()
Expand All @@ -387,6 +419,7 @@ def delete_prompt():
)
return {"status": "ok"}


@user.route("/api/update_prompt", methods=["POST"])
def update_prompt_name():
data = request.get_json()
Expand All @@ -396,27 +429,31 @@ def update_prompt_name():
# check if name is null
if name == "":
return {"status": "error"}
prompts_collection.update_one({"_id": ObjectId(id)},{"$set":{"name":name, "content": content}})
prompts_collection.update_one(
{"_id": ObjectId(id)}, {"$set": {"name": name, "content": content}}
)
return {"status": "ok"}



@user.route("/api/get_api_keys", methods=["GET"])
def get_api_keys():
user = "local"
keys = api_key_collection.find({"user": user})
list_keys = []
for key in keys:
list_keys.append({
"id": str(key["_id"]),
"name": key["name"],
"key": key["key"][:4] + "..." + key["key"][-4:],
"source": key["source"],
"prompt_id": key["prompt_id"],
"chunks": key["chunks"]
})
list_keys.append(
{
"id": str(key["_id"]),
"name": key["name"],
"key": key["key"][:4] + "..." + key["key"][-4:],
"source": key["source"],
"prompt_id": key["prompt_id"],
"chunks": key["chunks"],
}
)
return jsonify(list_keys)


@user.route("/api/create_api_key", methods=["POST"])
def create_api_key():
data = request.get_json()
Expand All @@ -433,12 +470,13 @@ def create_api_key():
"source": source,
"user": user,
"prompt_id": prompt_id,
"chunks": chunks
"chunks": chunks,
}
)
new_id = str(resp.inserted_id)
return {"id": new_id, "key": key}


@user.route("/api/delete_api_key", methods=["POST"])
def delete_api_key():
data = request.get_json()
Expand All @@ -449,4 +487,3 @@ def delete_api_key():
}
)
return {"status": "ok"}

Loading
0