add local-ai

podman/incubating/localai/README.md

@@ -0,0 +1,135 @@
+# Local AI with Anything LLM
+
+<https://github.com/Mintplex-Labs/anything-llm/blob/master/docker/HOW_TO_USE_DOCKER.md>
+
+<https://localai.io/>
+
+## Run locally
+
+```bash
+podman network create localai
+
+mkdir -p ./volumes/local-ai/local-ai
+mkdir -p ./volumes/local-ai/anythingllm
+
+# Local AI
+podman run \
+    -d \
+    -p 127.0.0.1:8080:8080 \
+    --network localai \
+    --name local-ai \
+    -v ./volumes/local-ai/local-ai:/build/models \
+    quay.io/go-skynet/local-ai:latest-cpu
+
+# Anything LLM Interface
+podman run -d \
+    -p 127.0.0.1:3001:3001 \
+    --cap-add SYS_ADMIN \
+    --network localai \
+    --name anything-llm \
+    -v ./volumes/local-ai/anythingllm:/app/server/storage \
+    -e STORAGE_DIR="/app/server/storage" \
+    docker.io/mintplexlabs/anythingllm
+```
+
+## Run in Production
+
+This installs both Local AI and Anything LLM as backend/frontend services.
+
+Note:
+
+You'll need folders for the models and anything-llm storage.
+
+```bash
+mkdir /models
+mkdir /anything-llm
+chown 1000:1000 /anything-llm
+```
+
+```bash
+podman network create localai
+
+# Local AI
+podman run \
+    -d \
+    -p 127.0.0.1:8080:8080 \
+    --network localai \
+    --name local-ai \
+    -v /models:/build/models \
+    quay.io/go-skynet/local-ai:latest-cpu
+
+# Anything LLM Interface
+podman run -d \
+    -p 127.0.0.1:3001:3001 \
+    --cap-add SYS_ADMIN \
+    --network localai \
+    --name anything-llm \
+    -v /anythingllm:/app/server/storage \
+    -e STORAGE_DIR="/app/server/storage" \
+    docker.io/mintplexlabs/anythingllm
+```
+
+## Models
+
+### Config
+
+```yaml
+name: llama-3.2
+parameters:
+  model: huggingface/Llama-3.2-3B-Instruct-f16.gguf
+  temperature: 0.6
+backend: llama-cpp
+# Default context size
+context_size: 8192
+threads: 16
+```
+
+### Chat
+
+llama-3.2-3b-instruct:q8_0
+
+### Code
+
+llama3.2-3b-enigma
+
+### Agent
+
+llama-3.2-3b-instruct:q8_0
+
+## Nginx
+
+```bash
+certbot-3 certonly --dns-route53 -d chatreesept.reeseapps.com
+```
+
+Make sure to add the following timeout configurations to your http block:
+
+```conf
+server {
+   # Enable websocket connections for agent protocol.
+   location ~* ^/api/agent-invocation/(.*) {
+      proxy_pass http://0.0.0.0:3001;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "Upgrade";
+   }
+
+   listen 80;
+   server_name [insert FQDN here];
+   location / {
+      # Prevent timeouts on long-running requests.
+      proxy_connect_timeout       605;
+      proxy_send_timeout          605;
+      proxy_read_timeout          605;
+      send_timeout                605;
+      keepalive_timeout           605;
+
+      # Enable readable HTTP Streaming for LLM streamed responses
+      proxy_buffering off; 
+      proxy_cache off;
+
+      # Proxy your locally running service
+      proxy_pass  http://0.0.0.0:3001;
+    }
+}
+```

podman/incubating/localai/chatreesept.reeseapps.com

@@ -0,0 +1,32 @@
+server {
+    listen 127.0.0.1:8443 ssl;
+    server_name chatreesept.reeseapps.com;
+
+    ssl_certificate /etc/letsencrypt/live/chatreesept.reeseapps.com/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/chatreesept.reeseapps.com/privkey.pem;
+    include /etc/letsencrypt/options-ssl-nginx.conf;
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+
+    location ~* ^/api/agent-invocation/(.*) {
+        proxy_pass http://localhost:3001;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "Upgrade";
+   }
+
+    location / {
+        client_max_body_size 50m;
+        # Prevent timeouts on long-running requests.
+        proxy_connect_timeout       605;
+        proxy_send_timeout          605;
+        proxy_read_timeout          605;
+        send_timeout                605;
+        keepalive_timeout           605;
+
+        # Enable readable HTTP Streaming for LLM streamed responses
+        proxy_buffering off; 
+        proxy_cache off;
+
+        proxy_pass http://localhost:3001;
+    }
+}