{ "default": { "hw": "m_24", "mode": "server", "variant": "default" }, "family": "hunyuan_v1_dense", "features": [ { "default": "off", "id": "api_key", "modes": [ "server", "server_tp", "disaggregated" ] }, { "default": "off", "id": "estimate_memory", "modes": [ "cli_run", "cli_generate" ] }, { "default": "off", "id": "kv_int8", "modes": [ "server", "cli_run", "cli_generate", "server_tp", "disaggregated" ] }, { "default": "off", "id": "lora", "modes": [ "server", "cli_run", "cli_generate", "server_tp", "disaggregated" ] }, { "default": "off", "id": "metrics", "modes": [ "server", "server_tp", "disaggregated" ] }, { "default": "on", "id": "tool_calling", "modes": [ "server", "cli_run", "cli_generate", "server_tp", "disaggregated" ] }, { "default": "off", "id": "turbo_kv", "modes": [ "server", "cli_run", "cli_generate", "server_tp", "disaggregated" ] } ], "i18n": { "ko": { "description": "Hunyuan-1.8B-Instruct-4bit: mlxcel의 텍스트 생성·API 서빙 명령을 구성하는 레시피입니다.", "title": "Hunyuan-1.8B-Instruct-4bit" } }, "model_id": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "modes": [ "server", "cli_run", "cli_generate", "server_tp", "disaggregated" ], "renderings": { "a100_80": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 72, "drafterGb": 0, "headroomGb": 5.76, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "a100_80", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "a100_80" } } ] }, "a100_80x2": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 144, "drafterGb": 0, "headroomGb": 11.52, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "a100_80x2", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "a100_80x2" } } ] }, "a100_80x4": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 288, "drafterGb": 0, "headroomGb": 23.04, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "a100_80x4", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "a100_80x4" } } ] }, "gb10_122": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-aarch64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-aarch64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-aarch64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-aarch64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 91.5, "drafterGb": 0, "headroomGb": 7.32, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "gb10_122", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [] }, "h100_80": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 72, "drafterGb": 0, "headroomGb": 5.76, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "h100_80", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "h100_80" } } ] }, "h100_80x2": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 144, "drafterGb": 0, "headroomGb": 11.52, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "h100_80x2", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "h100_80x2" } } ] }, "h100_80x8": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 576, "drafterGb": 0, "headroomGb": 46.08, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "h100_80x8", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "h100_80x8" } } ] }, "m1ultra_128": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 96, "drafterGb": 0, "headroomGb": 7.68, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m1ultra_128", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [] }, "m5max_128": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 96, "drafterGb": 0, "headroomGb": 7.68, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m5max_128", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [] }, "m_128": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 96, "drafterGb": 0, "headroomGb": 7.68, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_128", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_128" } } ] }, "m_16": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 10.72, "drafterGb": 0, "headroomGb": 1, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_16", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_16" } } ] }, "m_192": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 144, "drafterGb": 0, "headroomGb": 11.52, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_192", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_192" } } ] }, "m_24": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 16.08, "drafterGb": 0, "headroomGb": 1.29, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_24", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_24" } } ] }, "m_256": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 192, "drafterGb": 0, "headroomGb": 15.36, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_256", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_256" } } ] }, "m_32": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 21.44, "drafterGb": 0, "headroomGb": 1.72, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_32", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_32" } } ] }, "m_36": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 24.12, "drafterGb": 0, "headroomGb": 1.93, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_36", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_36" } } ] }, "m_48": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 36, "drafterGb": 0, "headroomGb": 2.88, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_48", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_48" } } ] }, "m_512": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 384, "drafterGb": 0, "headroomGb": 30.72, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_512", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_512" } } ] }, "m_64": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 48, "drafterGb": 0, "headroomGb": 3.84, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_64", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_64" } } ] }, "m_96": { "backend": "metal", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-macos-aarch64.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-macos-aarch64:$PATH\" \\\n && mlxcel --version", "title": "macOS archive" }, { "argv": [ "sh", "-lc", "brew tap lablup/tap && brew install mlxcel" ], "id": "brew", "text": "brew tap lablup/tap && brew install mlxcel", "title": "Homebrew" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-macos", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features metal,accelerate && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (macOS)" }, { "argv": [ "brew", "install", "mlxcel" ], "id": "brew-command", "text": "brew install mlxcel", "title": "Brew install" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 72, "drafterGb": 0, "headroomGb": 5.76, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "m_96", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "m_96" } } ] }, "rtx_4090_24": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 21.6, "drafterGb": 0, "headroomGb": 1.73, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "rtx_4090_24", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "rtx_4090_24" } } ] }, "rtx_4090_24x2": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 43.2, "drafterGb": 0, "headroomGb": 3.46, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "rtx_4090_24x2", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "rtx_4090_24x2" } } ] }, "rtx_5090_32": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 28.8, "drafterGb": 0, "headroomGb": 2.3, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "rtx_5090_32", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [ { "code": "not_measured_on_tier", "params": { "hw": "rtx_5090_32" } } ] }, "v100_32": { "backend": "cuda", "commands": { "download": { "argv": [ "mlxcel", "download", "mlx-community/Hunyuan-1.8B-Instruct-4bit" ], "id": "download", "text": "mlxcel download mlx-community/Hunyuan-1.8B-Instruct-4bit", "title": "Download checkpoint" }, "install": [ { "argv": [ "sh", "-lc", "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version" ], "id": "archive", "text": "MLXCEL_INSTALL_DIR=\"${MLXCEL_INSTALL_DIR:-$HOME/.local/share/mlxcel}\" \\\n && mkdir -p \"$MLXCEL_INSTALL_DIR\" \\\n && curl -fL -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" https://github.com/lablup/mlxcel/releases/download/v0.8.1/mlxcel-linux-x86_64-cuda13.zip \\\n && unzip -o \"$MLXCEL_INSTALL_DIR/mlxcel.zip\" -d \"$MLXCEL_INSTALL_DIR\" \\\n && export PATH=\"$MLXCEL_INSTALL_DIR/mlxcel-linux-x86_64-cuda13/bin:$PATH\" \\\n && mlxcel --version", "title": "Linux CUDA archive" }, { "argv": [ "sh", "-lc", "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"" ], "id": "source-linux", "text": "git clone https://github.com/lablup/mlxcel.git && cd mlxcel && cargo build --release --features cuda && export PATH=\"$PWD/target/release:$PATH\"", "title": "Build from source (Linux)" } ], "run": [ { "argv": [ "mlxcel-server", "-m", "mlx-community/Hunyuan-1.8B-Instruct-4bit", "--alias", "hunyuan-1.8b-instruct-4bit", "--ctx-size", "8192", "--n-parallel", "4" ], "env": { "LLAMA_ARG_ALIAS": "hunyuan-1.8b-instruct-4bit", "LLAMA_ARG_CTX_SIZE": "8192", "LLAMA_ARG_MODEL": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "LLAMA_ARG_N_PARALLEL": "4" }, "id": "server", "text": "mlxcel-server -m mlx-community/Hunyuan-1.8B-Instruct-4bit --alias hunyuan-1.8b-instruct-4bit \\\n --ctx-size 8192 --n-parallel 4", "title": "mlxcel-server" } ] }, "envText": "LLAMA_ARG_MODEL=mlx-community/Hunyuan-1.8B-Instruct-4bit LLAMA_ARG_ALIAS=hunyuan-1.8b-instruct-4bit LLAMA_ARG_CTX_SIZE=8192 LLAMA_ARG_N_PARALLEL=4 \\\n mlxcel-server", "expect": { "kind": "none" }, "features": [ "tool_calling" ], "fit": { "alternatives": [], "budgetGb": 28.8, "drafterGb": 0, "headroomGb": 2.3, "kvGb": 0.5, "source": "approx", "state": "fits", "weightsGb": 0.94 }, "hw": "v100_32", "mode": "server", "snippets": { "anthropic": "curl http://127.0.0.1:8080/v1/messages \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"max_tokens\":128,\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "curl": "curl http://127.0.0.1:8080/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "javascript": "import OpenAI from \"openai\";\n\nconst client = new OpenAI({ baseURL: \"http://127.0.0.1:8080/v1\", apiKey: \"not-needed\" });\nconsole.log(await client.chat.completions.create({ model: \"hunyuan-1.8b-instruct-4bit\", messages: [{ role: \"user\", content: \"Hello\" }] }));", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"not-needed\")\nprint(client.chat.completions.create(model=\"hunyuan-1.8b-instruct-4bit\", messages=[{\"role\": \"user\", \"content\": \"Hello\"}]))", "responses": "curl http://127.0.0.1:8080/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\":\"hunyuan-1.8b-instruct-4bit\",\"input\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'" }, "variant": "default", "warnings": [] } }, "schema_version": 1, "variants": [ { "backends": [ "cuda", "metal" ], "gated": false, "id": "default", "model_id": "mlx-community/Hunyuan-1.8B-Instruct-4bit", "precision": "q4" } ] }