LLM Microservice Call

Ever wanted to make an LLM call on a segment? Redshred has a feature that allows you to do this called, the “LLM Microservice”. For any segment that has been created, you can pass the segment link to the LLM microservice, along with the LLM prompt, and get back the response from the LLM. There is additional support to pass the image crop of the segment to the LLM.

Query Endpoints

The LLM Microservice endpoint is:

https://api.staging.redshred.com/v2/collections/{collection_name}/services/llm-query

Sample API Call to invoke the endpoint:

# Set up environment variables
export REDSHRED_HOST="https://api.staging.redshred.com"
export COLLECTION="tracee"
export REDSHRED_TOKEN="<token>"
export SEGMENT_ID="HYSiSRBdFAsugxh5aWEeik"

# Create a temporary file with the JSON payload
cat > llm_request.json << 'EOL'
{
  "collection": "$COLLECTION",
  "segment_id": "$SEGMENT_ID",
  "prompt": "Please extract the formula from the attached image in LaTeX format.",
  "model_config": {
    "model": "gpt-4o-mini",
    "temperature": 0.5,
    "max_tokens": 256
  },
  "segment_type": "llm_extract",
  "image_attachment": "iVBORw0KGgoAAAANSUhEUgAAAisAAAAdCAIAAADkcLvmAAANZklEQVR4nOzdeVAT598A8CdpIDYxGCglgLWAIIXBiorWCtbWgpRj6tFikaZQLI6ABYoCLSCdpiOXttVaR2DwCEigNEOhtCVEpMfQ4VCOSQQJyCH3Ga5whSRk36nPO/uLCVQEi619Pn/tfvfZfZ4sM/uwz7UkDMMAgiAIgiw74uMuAIIgCPIfRVLdUSgUUqlUMxGZTNbS0sIwrKamxsjIyNjYeBlLuFDd3d0ikYhGo23evFlLSwuPNzY2KpVKa2vrRV8Zw7DJyUnNOIlEWrFihVQqVSgUmkefeuqpp59+GgAwOTlZW1v74osvUqnURZcBQRDkyXPfO1BPT8+xY8doNJqvry+LxYqKitq/fz+NRisoKBgYGIiNjbW0tGSxWI+vtHO7efOmp6cnl8slEonl5eVWVlbFxcUAAKVSGRsbSyKRkpKSZmZmFn39iYkJNptNo9EcHR1ZLFZMTIyfn9+zzz4bGRkJAKirq3NycqLRaNHR0SwW67PPPgsJCTEwMDhy5AgAoKKigsPhGBkZnTlz5pH+aARBkH8/7H5paWlEInFwcBCP+Pr6Dg8PM5nMvr4+DMOmp6exf5LU1NT169f39vbikYsXL+ro6PT19V24cCE3N/eRlLmtrQ0A8P333+ORK1eu/PDDD3CbyWRaWVmppudyuZmZmTKZzN3dHcOw2dnZmZmZJZYBQRDkCaPeD1RWVmZnZ6evr49HAgMDdXV1W1paGAwGbFxa9lpyXtnZ2SEhIVwu19DQEA/u27dPIpFkZWVdu3Ztx44dj6TMZWVlJBLJyckJjzg4OLz66qtwu7S01N3dXTW9mZmZs7NzXV2diYnJn2+a9yyxDAiCIE8Y9ccij8dzdXWF2yUlJQCAl156CQDAYDCSkpLy8vLGxsYeRznn0NHR4e/vHxQUpNbHQyaTAQDNzc329vYJCQk5OTl3796d8wpisTgiIkKpVOKRO3fuxMXFaaYsKCiwt7fX0dEBAJSXl8vlcktLSzqdDgCor69va2vDa6CioiIAwJYtW/T19c3MzIRCIYfD4fF4j/rXIwiC/OvdNxJBKBR2dXXZ2dm1tbX98ccfra2tO3fuhIdyc3PFYrGBgcEylKmjo0MikWjGKRTK2rVr8d3Y2Njx8fHQ0FDN0wEAurq6n3zyiVgsptPpJBJpzoz09fWpVKqfn9/ly5eJROKdO3cOHjyYmZmplkypVPL5/MOHD3d1dTU1NV26dEk1TUFBAYVCsbCw6OzsrKio4PF4zs7O8BCdTv/1118nJib09PSWcD8QBEGeTPc9mnk83qpVq2pqaioqKjIyMq5evYofIhKJy1P9AADy8/OFQqFm3NzcPCoqCm7LZLJvv/1227Ztq1evVksGhyG8/PLLsI7567xY93zwwQeRkZFeXl6ZmZmao+Zu3LgxNDQklUpTU1OvX7+OVzAQj8ezsrK6fPny9PQ0l8uNjo5WPaqtrY2qHwRBkLmpdgo5ODgcOHAAbp85c+af3HleXV0NAAgLC9M8ZG9vb2ho+FCFDw4OplKpQqFwzqMnTpxgMBhKpRLDsJKSktLSUvzQ2NiYlpbWxYsX4S6Hw+no6Hj4X4MgCPIvNjAwMDIyohlXKBTwyYlhWEtLC76N+18/0MjISEVFhZubG9z18PDQ1taenZ1d/kUTent7m+fS1dWFpxkfHwcAaL4AlZaWlpWVsVgsbW3tBWbX0tJSUlLi5eV1/vz5OX8sj8dzcXEhEAgAAAsLi23btsG5U7DXRy6Xu7i4wJQ7d+5cs2bNEn46giDIvwyHw0lLS1u5cqVaXCgUUiiU/Px8uEsikQIDA1tbW1XT/K8Vjs/nK5VK/GEKn6SnT5+OiIiYrx/lb5Kenl5VVaUZt7S0jI+Px7cJBIJad9HMzExgYKCrq6u/v/8C82ptbfXw8EhPT7e1tT1x4kRAQEBKSgqsbKDe3l6BQACn/gAAjIyMAAA//fSTiYnJhg0beDyejY3Nc889B4+i6gdBkP+UU6dOyWSyTz/9VC0ulUqZTKZMJsMjzz//fGJi4p49e5KTk21sbGCQgP/X7+npefv27bq6OriLYdj58+cbGhqSkpLmy7ujo+PmzZtUKlWhUFhaWpLJ5KKiIlNTU2dn54qKCoFAsH37dltb2/Hx8eLiYm1tbYVCYWhoCN8hoOnpaTKZvLiRyp6envX19UKhEJ6uUCjee++9kZERLpe7atWqOU9Ry04sFjs7O7PZbFtbWxiJioqanZ09ffo0fkpycvLRo0cHBwfxLqXff/89Ojq6rKxMLpcbGxu/++67586de6iSV1VVdXZ2amlpyWQyOGhbIBDs2LHD2tr6559/FovFrq6uxsbGarf3hRdewK8wOTmJVlhAEOTx+vHHH2NiYgQCgeYzPDQ0tLGxkc/n5+Xl7du3D49XVVUxmcwbN27AscR/1jTd3d1ffvmllpYWlUoNCQk5fvy4t7c3HHVWXFw8X8Nfenr6F198AZvpUlJSRCKRUqk0NzeH0z8FAoGjoyOGYVVVVR9++KFEIsEwTCQSpaSkqF6EzWZXVlYuruVRIpHs3bvXzc0tKysrNTV1//7933zzjVwu/4tTNLPr6upSS4NHZDJZeno6bOjz9vYOCwvz8/OD1WdMTEx1dfWhQ4fgkIdffvllgWVWKpURERGFhYVw99ixY3CtIzc3Nxj5+uuvT548OeftxS+iUChCQkIWmCOCIMjfYXx8nMFgXLlyRfPQ9evXvby8MjIyAAB5eXlqR52cnIKCguC2+poIC1RZWenq6orv9vT0wNEBPj4+MHL27NmkpKSxsbFNmzbB6kc1JTQ1NXXr1q3FFQCanZ0NDQ1ls9mdnZ0PTLz07Jbu7NmzcXFx+C68G1999RX+V9y7d299ff2ctxfX0dGhugYEgiDI8ktMTCQQCP39/WrxoaGh7du3j4yMzFcDnTt3jkwmw//1F9nBk5CQ4OfnBwCYmpqqra3FMIxIJBYWFuro6OTk5AAAMjIycnNz09PTHRwcaDQafPlSKBTPPPMMfpH+/v7w8PBr164trgyjo6PHjx/38PBwdHSEs1Chpqam0dHRrVu3qqVfYnZLh2FYXFxcQ0MDAKCvrw+u9EOn0/l8/ltvvZWTk6NQKFpaWqytrd9++23N2wvXpJiZmYmLi/P29lZdBgJBEGSZpaWlmZmZac7SOXr0aHx8/P83ss1l69atMzMz2dnZYWFhi1wqprm5GTbTUSiUsrKyrKwsAwODoqIiFovl4eHh7u5OJBJNTEzwZHBpnKCgILhKDdTe3v7GG28sugAHDx7k8/nu7u4rV67csmWLn59fZGSkv79/ZWWlZvWzxOweidHRUZlMButgBoORmJjY3d2tVColEklAQICHhweDwYAr/cx5e+FFyGTywMAAnO2EIAjyWHR2djY0NFhZWanFr169ampq+tprr/3FuevWrQMAFBYWqs9IXbiNGzc2Nzdv2rQJAFBbWxsQEDA6OkogEODj9bffftu1axdMho9qq62tfeedd1QHSRcUFLz//vuLK4C5uTmfzwcADA8PV1dXC4XCycnJtWvXuru7zzcDdCnZPRJ0Ol1PT6+/v5/BYBAIhPb29j179hQWFu7evRsmKCwshKPhNW8vPjxPIpFQKBS0yhyCII/RrVu3AABqL0Dt7e1sNvuB7Uz6+vorVqyAo94W3woXEhJCJBIpFIpYLLaxsYmLi6NSqePj41NTU8nJyRs2bAAA+Pj4lJaWXrhwwcbGhs/nf/755wAALpebm5ubnZ0NezvwYXkPBX8i6+np7b7ngacsJbtHgkAgpKWlhYeHe3t7d3d379q1a2hoKCkpCY5uEIlE3333HVz8VO32wle6jz76aN26ddbW1gQCobe395/5lSYEQf4LBgcHAQCqc4CUSuXhw4cTEhKm7oGdCHDg7ujoqI6Ojur/zRQKZXBwcHZ2lrCUCac9PT1EIvGBHRIjIyMSiWTNmjWwBDKZLC0t7ciRIxMTE5qTmP4+y5zdfBQKRVdXl66u7nxDxnFqt1cikeTn5zOZTKlUSqFQlqWwCIIgc+BwON7e3gEBAcnJyTBy+/bt9evXz5deJBKpNtkZGhr29/fL5fIlTTVd4L/huvfgu729vbCtaZnrg39C9QMnBpuami4kpdrtbWpqOnDgAHwx+ttKhyAI8mBwYBR8y4FWr17NZrNV05SXl6empgYHB2/evBnO5cdNTEzANaOXdbEDSHUwArJwdnZ2j7sICIIgf4LdGT09PXiETqf7+vqqpiGRSKmpqa+//rrqjFS4MsDk5OQrr7wyx/eBEARBEOSvGRsbb9y4saWlZRHnikQiAAD8yACqgRAEQZCH5uvre/fuXbFY/LAnVlZWkkgkJpN537pwCIIgCLJAUqnUwsIiPj7ex8fnoU588803GQzGpUuXUA2EIAiCLBKfzw8PD6+pqVn413AaGxt3794tEAjgxE3UCocgCIIshouLy6FDh06ePLnA9HK5/OOPP87Ly8PXDUDvQAiCIMjicTic4eHh4OBg1S+raRoeHj516pSPj4/qsgCoBkIQBEGWpL+/X1tbW3Xep6bGxkZzc3O1KUD/FwAA//+pdLNMcaI5/wAAAABJRU5ErkJggg=="
}
EOL

# Make the POST request
curl -X POST \
  "$REDSHRED_HOST/v2/collections/$COLLECTION/services/llm-query" \
  -H "Authorization: Token $REDSHRED_TOKEN" \
  -H "Content-Type: application/json" \
  -d @llm_request.json \
  --output llm_response.json

# Clean up
rm llm_request.json

# TODO: Add code for python example

There is a complete runnable example of this code in random-go.

The code queries for segments of type formula and prompts the LLM to extract the formula from the attached image in LaTeX format. It relies on the tracee collection on api.staging.redshred.com.

package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"github.com/redshred/redshred-client-go/pkg/redshred"
)

// Model config as a package-level constant
var defaultModelCfg = ModelConfig{
	Model:       "gpt-4o-mini",
	Temperature: 0.5,
	MaxTokens:   256,
}

type (
	LLMQueryRequest struct {
		Collection     string      `json:"collection"`
		SegmentID      string      `json:"segment_id"`
		Prompt         string      `json:"prompt"`
		ModelConfig    ModelConfig `json:"model_config"`
		SegmentType    string      `json:"segment_type"`
		ImageAttachment string     `json:"image_attachment,omitempty"`
	}

	ModelConfig struct {
		Model       string  `json:"model"`
		Temperature float64 `json:"temperature"`
		MaxTokens   int     `json:"max_tokens"`
	}

	LLMQueryResponse struct {
		Prediction  string `json:"prediction"`
		SegmentLink string `json:"segment_link"`
		RawOutput   string `json:"raw_output"`
	}
)

// processSegment processes a single segment, downloading its crop and sending to LLM
func processSegment(rs redshred.RedShredClient, coll *redshred.Collection, segment redshred.Segment) error {
	// Construct the crop image URL
	imageURL := fmt.Sprintf("%s/v2/collections/%s/services/crop_by_link?segment_link=%s",
		rs.BaseUrl, coll.Name, segment.SelfLink)

	// Download the crop image
	imgData, err := DownloadImage(&rs, imageURL)
	if err != nil {
		return fmt.Errorf("downloading crop image: %w", err)
	}

	// Process the image with LLM
	if err := processImage(rs, coll, segment, imgData); err != nil {
		return fmt.Errorf("processing image: %w", err)
	}

	return nil
}

func processImage(rs redshred.RedShredClient, coll *redshred.Collection, segment redshred.Segment, imgData []byte) error {
	reqBody := LLMQueryRequest{
		Collection:     coll.Name,
		SegmentID:      segment.ID,
		Prompt:         "Please extract the formula from the attached image in LaTeX format.",
		ModelConfig:    defaultModelCfg,
		SegmentType:    "llm_extract",
		ImageAttachment: base64.StdEncoding.EncodeToString(imgData),
	}

	reqJSON, err := json.Marshal(reqBody)
	if err != nil {
		return fmt.Errorf("marshaling request: %w", err)
	}

	url := fmt.Sprintf("%s/v2/collections/%s/services/llm_query", rs.BaseUrl, coll.Name)
	resp, err := rs.JSONRequest("POST", url, bytes.NewReader(reqJSON))
	if err != nil {
		return fmt.Errorf("LLM query: %w", err)
	}

	body, err := io.ReadAll(bytes.NewReader(resp))
	if err != nil {
		return fmt.Errorf("reading response: %w", err)
	}

	var llmResp LLMQueryResponse
	if err := json.Unmarshal(body, &llmResp); err != nil {
		return fmt.Errorf("parsing response: %w", err)
	}

	if llmResp.RawOutput == "" {
		return fmt.Errorf("empty response, prediction: %s", llmResp.Prediction)
	}

	fmt.Printf("\nExtracted formula from %s:\n%s\nSegment Link: %s\n",
		segment.ID, llmResp.RawOutput, llmResp.SegmentLink)
	return nil
}