8000 New `InferenceClient` endpoint type by nsarrazin · Pull Request #1813 · huggingface/chat-ui · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

New InferenceClient endpoint type #1813

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 15 commits into
base: main
Choose a base branch
from
+1,108 −288
Draft

New InferenceClient endpoint type #1813

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
8881f1e
work on hf inference endpoint
nsarrazin Apr 30, 2025
332bf33
working tools
nsarrazin Apr 30, 2025
b4dc31e
fix: debug tool
nsarrazin Apr 30, 2025
4747309
feat: working tools with new endpoint type
nsarrazin Apr 30, 2025
584ad18
feat: move config to HfInference
nsarrazin Apr 30, 2025
57d9eb9
Merge branch 'main' into feat/hfinference_endpoint
nsarrazin May 1, 2025
7e5beeb
wip
nsarrazin May 5, 2025
7579ee4
fix: hf inference endpoint
nsarrazin May 6, 2025
5a4785c
Merge branch 'main' into feat/hfinference_endpoint
nsarrazin May 6, 2025
87419d6
refactor: rename endpoint
nsarrazin May 6, 2025
1a02981
fix: rendering
nsarrazin May 6, 2025
412036d
Merge branch 'feat/hfinference_endpoint' of github.com:huggingface/ch…
nsarrazin May 6, 2025
0f056b3
Merge branch 'main' into feat/hfinference_endpoint
nsarrazin May 7, 2025
61a322c
feat: working tools on every supported model
nsarrazin May 12, 2025
86d0097
fix: multimodal
nsarrazin May 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 39 additions & 31 deletions chart/env/prod.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ envVars:
"max_new_tokens": 1024,
"truncate": 7167
},
"endpoints": [{"type" : "inference-providers"}],
"promptExamples": [
{
"title": "Write an email",
Expand Down Expand Up @@ -91,6 +92,7 @@ envVars:
"stop": ["<|endoftext|>", "<|im_end|>"],
"temperature": 0.6,
},
"tools": true,
"promptExamples": [
{
"title": "Write an email",
Expand All @@ -107,7 +109,7 @@ envVars:
],
"endpoints": [
{
"type": "openai",
"type": "inference-providers",
"baseURL": "https://api-inference.endpoints.huggingface.tech/models/Qwen/Qwen3-235B-A22B/v1"
}
]
Expand All @@ -126,6 +128,7 @@ envVars:
"max_new_tokens": 3072
},
"tools": true,
"endpoints": [{"type" : "inference-providers"}],
"promptExamples": [
{
"title": "Write an email",
Expand All @@ -148,12 +151,14 @@ envVars:
"websiteUrl": "https://docs.cohere.com/docs/command-r-plus",
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/cohere-logo.png",
"tools": true,
"systemRoleSupported": false,
"parameters": {
"stop": ["<|END_OF_TURN_TOKEN|>", "<|im_end|>"],
"truncate": 28672,
"max_new_tokens": 2048,
"temperature": 0.3
},
"endpoints": [{"type" : "inference-providers"}],
"promptExamples": [
{
"title": "Generate image",
Expand All @@ -180,6 +185,7 @@ envVars:
"beginToken": "",
"endToken": "</think>"
},
"tools": true,
"promptExamples": [
{
"title": "Rs in strawberry",
Expand All @@ -196,8 +202,8 @@ envVars:
],
"endpoints": [
{
"type": "openai",
"baseURL": "https://api-inference.endpoints.huggingface.tech/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B/v1"
"type": "inference-providers",
"baseURL": "https://internal.api-inference.huggingface.co/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B/v1"
}
]
},
Expand Down Expand Up @@ -229,13 +235,14 @@ envVars:
],
"endpoints": [
{
"type": "openai",
"type": "inference-providers",
"baseURL": "https://api-inference.endpoints.huggingface.tech/models/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF/v1"
}
]
},
{
"name": "Qwen/QwQ-32B",
"tools": true,
"preprompt": "You are a helpful and harmless assistant. You are Qwen developed by Alibaba. You should think step-by-step.",
"modelUrl": "https://huggingface.co/Qwen/QwQ-32B",
"websiteUrl": "https://qwenlm.github.io/blog/qwq-32b/",
Expand All @@ -262,15 +269,16 @@ envVars:
],
"endpoints": [
{
"type": "openai",
"baseURL": "https://atv7xs1nxxtx2wl0.us-east-1.aws.endpoints.huggingface.cloud/v1"
"type": "inference-providers",
}
]
},
{
"name": "google/gemma-3-27b-it",
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/google-logo.png",
"multimodal": true,
"tools": true,
"systemRoleSupported" : false,
"description": "Google's latest open model with great multilingual performance, supports image inputs natively.",
"websiteUrl": "https://blog.google/technology/developers/gemma-3/",
"promptExamples": [
Expand All @@ -289,22 +297,23 @@ envVars:
],
"endpoints": [
{
"type": "openai",
"type": "inference-providers",
"baseURL": "https://wp0d3hn6s3k8jk22.us-east-1.aws.endpoints.huggingface.cloud/v1",
"multimodal": {
"image": {
"maxSizeInMB": 10,
"maxWidth": 560,
"maxHeight": 560,
"supportedMimeTypes": ["image/jpeg"],
"preferredMimeType": "image/jpeg"
"supportedMimeTypes": ["image/jpeg", "image/png", "image/webp"],
"preferredMimeType": "image/webp"
}
}
}
]
},
{
"name": "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
"tools": true,
"displayName": "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
"description": "A small model with good capabilities in language understanding and commonsense reasoning.",
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/mistral-logo.png",
Expand All @@ -328,8 +337,8 @@ envVars:

"endpoints": [
{
"type": "openai",
"baseURL": "https://hkjfqcryevvq9cie.us-east-1.aws.endpoints.huggingface.cloud/v1"
"type": "inference-providers",
"baseURL": "https://hkjfqcryevvq9cie.us-east-1.aws.endpoints.huggingface.cloud/v1"
}
]
},
Expand All @@ -340,6 +349,7 @@ envVars:
"websiteUrl": "https://qwenlm.github.io/blog/qwen2.5-vl/",
"modelUrl": "https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct",
"multimodal": true,
"tools": true,
"promptExamples": [
{
"title": "Write an email",
Expand All @@ -356,8 +366,7 @@ envVars:
],
"endpoints": [
{
"type": "openai",
"baseURL": "https://lf91qeosuambouj4.us-east-1.aws.endpoints.huggingface.cloud/v1",
"type": "inference-providers",
"multimodal": {
"image": {
"maxSizeInMB": 10,
Expand All @@ -372,6 +381,8 @@ envVars:
},
{
"name": "microsoft/Phi-4",
"tools": true,
"systemRoleSupported": false,
"description": "One of the best small models, super fast for simple tasks.",
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/microsoft-logo.png",
"modelUrl": "https://huggingface.co/microsoft/Phi-4",
Expand Down Expand Up @@ -399,14 +410,15 @@ envVars:
],
"endpoints": [
{
"type": "openai",
"type": "inference-providers",
"baseURL": "https://up5ijetg6a2e9zlb.us-east-1.aws.endpoints.huggingface.cloud/v1"
}
]
},
{
"name": "NousResearch/Hermes-3-Llama-3.1-8B",
"description": "Nous Research's latest Hermes 3 release in 8B size. Follows instruction closely.",
"tools": true,
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/nous-logo.png",
"websiteUrl": "https://nousresearch.com/",
"modelUrl": "https://huggingface.co/NousResearch/Hermes-3-Llama-3.1-8B",
Expand All @@ -429,23 +441,8 @@ envVars:
"temperature": 0.6,
"truncate": 14336,
"max_new_tokens": 1536
}
},
{
"name": "internal/task",
"tokenizer" : "NousResearch/Hermes-3-Llama-3.1-8B",
"unlisted": true,
"tools" : true,
"endpoints": [
{
"type": "openai",
"baseURL": "https://api-inference.endpoints.huggingface.tech/models/NousResearch/Hermes-3-Llama-3.1-8B/v1"
}
],
"parameters": {
"temperature": 0.1,
"max_new_tokens": 256
},
"endpoints": [{"type" : "inference-providers"}]
}
]

Expand Down Expand Up @@ -539,7 +536,18 @@ envVars:
PUBLIC_APP_DISCLAIMER: 1
PUBLIC_PLAUSIBLE_SCRIPT_URL: "/js/script.js"
REQUIRE_FEATURED_ASSISTANTS: "true"
TASK_MODEL: "internal/task"
TASK_MODEL: >
{
"name": "NousResearch/Hermes-3-Llama-3.1-8B",
"unlisted": true,
"endpoints": [{"type" : "inference-providers"}],
"parameters": {
"temperature": 0.1,
"max_new_tokens": 256
}
}


TEXT_EMBEDDING_MODELS: >
[{
"name": "bge-base-en-v1-5-sxa",
Expand Down
Loading
0