Local AI with Anything LLM

https://github.com/Mintplex-Labs/anything-llm/blob/master/docker/HOW_TO_USE_DOCKER.md

https://localai.io/

Run locally

podman network create localai

mkdir -p ./volumes/local-ai/local-ai
mkdir -p ./volumes/local-ai/anythingllm

# Local AI
podman run \
    -d \
    -p 127.0.0.1:8080:8080 \
    --network localai \
    --name local-ai \
    -v ./volumes/local-ai/local-ai:/build/models \
    quay.io/go-skynet/local-ai:latest-cpu

# Anything LLM Interface
podman run -d \
    -p 127.0.0.1:3001:3001 \
    --cap-add SYS_ADMIN \
    --network localai \
    --name anything-llm \
    -v ./volumes/local-ai/anythingllm:/app/server/storage \
    -e STORAGE_DIR="/app/server/storage" \
    docker.io/mintplexlabs/anythingllm

Run in Production

This installs both Local AI and Anything LLM as backend/frontend services.

Note:

You'll need folders for the models and anything-llm storage.

mkdir /models
mkdir /anything-llm
chown 1000:1000 /anything-llm

podman network create localai

# Local AI
podman run \
    -d \
    -p 127.0.0.1:8080:8080 \
    --network localai \
    --name local-ai \
    -v /models:/build/models \
    quay.io/go-skynet/local-ai:latest-cpu

# Anything LLM Interface
podman run -d \
    -p 127.0.0.1:3001:3001 \
    --cap-add SYS_ADMIN \
    --network localai \
    --name anything-llm \
    -v /anythingllm:/app/server/storage \
    -e STORAGE_DIR="/app/server/storage" \
    docker.io/mintplexlabs/anythingllm

Models

Config

name: llama-3.2
parameters:
  model: huggingface/Llama-3.2-3B-Instruct-f16.gguf
  temperature: 0.6
backend: llama-cpp
# Default context size
context_size: 8192
threads: 16

Chat

llama-3.2-3b-instruct:q8_0

Code

llama3.2-3b-enigma

Agent

llama-3.2-3b-instruct:q8_0

Nginx

certbot-3 certonly --dns-route53 -d chatreesept.reeseapps.com

Make sure to add the following timeout configurations to your http block:

server {
   # Enable websocket connections for agent protocol.
   location ~* ^/api/agent-invocation/(.*) {
      proxy_pass http://0.0.0.0:3001;
      proxy_http_version 1.1;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection "Upgrade";
   }

   listen 80;
   server_name [insert FQDN here];
   location / {
      # Prevent timeouts on long-running requests.
      proxy_connect_timeout       605;
      proxy_send_timeout          605;
      proxy_read_timeout          605;
      send_timeout                605;
      keepalive_timeout           605;

      # Enable readable HTTP Streaming for LLM streamed responses
      proxy_buffering off; 
      proxy_cache off;

      # Proxy your locally running service
      proxy_pass  http://0.0.0.0:3001;
    }
}