diff --git a/README.md b/README.md index 86e42df3..adc082cd 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ ./scripts/dev.sh ``` -脚本会自动启动 MySQL、Redis、MinIO,首次初始化数据库结构,并同时启动后端、主站前后台、分销独立后台与分销专属前台。 +脚本会自动启动 MySQL、Redis、MinIO、本地 OCR sidecar,首次初始化数据库结构,并同时启动后端、主站前后台、分销独立后台与分销专属前台。 - 主站前台地址:`http://localhost:5189` - 主站后台地址:`http://localhost:5189/admin/login`(超级管理员默认 `admin` / `admin123456`) @@ -24,6 +24,7 @@ - 解忧商行后台:`https:////login` - 分销管理后台:`http://localhost:5190`(在主站后台“分销站点”开通站点并配置分销操作员登录) - 分销专属前台:`http://localhost:5191`(独立端口隔离,支持 Host 域名与 X-Distribution-Site 头识别站点) +- OCR sidecar:`http://127.0.0.1:18787`(仅本地回环地址,二维码群名识别使用) > 生产环境不使用上述端口:Caddy 按域名分流,主站和解忧商行后台分别通过各自经营域名进入,分销管理后台默认使用 `https://distribution.`,可由 `DISTRIBUTION_ADMIN_DOMAIN` 覆盖;平台运营统计后台使用 `OPS_DOMAIN`,未配置时默认 `https://ops.`(见 `deploy/caddy/Caddyfile`)。主站后台的「打开独立分销后台」与开通凭证通过构建期 `VITE_DISTRIBUTION_ADMIN_URL` 注入该地址;未配置时按 `DISTRIBUTION_ADMIN_DOMAIN` 推导。 diff --git a/backend/.env.example b/backend/.env.example index 625fdb88..11c68eb2 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -26,6 +26,8 @@ CADDY_DOMAIN= TENANT_JIEYOU_DOMAIN= OPS_DOMAIN=ops.localhost DISTRIBUTION_ADMIN_DOMAIN=distribution.localhost +# 本地 OCR sidecar;留空时回退到后台配置的远程 PaddleOCR API。 +OCR_SERVICE_URL=http://127.0.0.1:18787 # 前端后台页面入口;本地默认 /admin,生产建议改成不公开的单段路径。 VITE_ADMIN_BASE_PATH=/admin diff --git a/backend/.env.prod.example b/backend/.env.prod.example index cac4c4ee..9ca1e561 100644 --- a/backend/.env.prod.example +++ b/backend/.env.prod.example @@ -13,6 +13,8 @@ OPS_DOMAIN= # 分销管理后台域名;留空时生产环境按 distribution. 推导。 # Caddy、前端构建和后端域名白名单必须使用同一个域名。 # DISTRIBUTION_ADMIN_DOMAIN=distribution.hfb.221329.cc.cd +# OCR sidecar,生产 Compose 默认值为 http://ocr:8787。 +# OCR_SERVICE_URL=http://ocr:8787 # 选号网(hfb_show)独立域名,与主站同机;不部署选号网可留空。 # CADDY_SHOW_DOMAIN=show.example.com # 选号网源码目录(相对 hfb_sys 根或绝对路径),默认 ../hfb_show diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index fe32a661..0ac24ab5 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -21,6 +21,7 @@ type Config struct { FieldEncryptionLegacyKey string ExternalUploadSecret string ExternalUploadAllowedIPs []string + OCRServiceURL string BootstrapAdminUsername string BootstrapAdminPassword string BootstrapAdminNickname string @@ -105,6 +106,7 @@ func Load() Config { FieldEncryptionLegacyKey: fieldEncryptionLegacyKey(getEnv("APP_ENV", "development")), ExternalUploadSecret: getEnv("EXTERNAL_UPLOAD_SECRET", ""), ExternalUploadAllowedIPs: getEnvList("EXTERNAL_UPLOAD_ALLOWED_IPS"), + OCRServiceURL: strings.TrimRight(strings.TrimSpace(getEnv("OCR_SERVICE_URL", "")), "/"), BootstrapAdminUsername: getEnv("ADMIN_BOOTSTRAP_USERNAME", ""), BootstrapAdminPassword: getEnv("ADMIN_BOOTSTRAP_PASSWORD", ""), BootstrapAdminNickname: getEnv("ADMIN_BOOTSTRAP_NICKNAME", "超级管理员"), diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go index 79d512c3..7252c5fd 100644 --- a/backend/internal/config/config_test.go +++ b/backend/internal/config/config_test.go @@ -111,6 +111,13 @@ func TestLoadDatabaseConfigInvalidValuesUseDefaults(t *testing.T) { } } +func TestLoadOCRServiceURL(t *testing.T) { + t.Setenv("OCR_SERVICE_URL", " http://ocr:8787/ ") + if got := Load().OCRServiceURL; got != "http://ocr:8787" { + t.Fatalf("OCRServiceURL = %q, want http://ocr:8787", got) + } +} + // TestFieldEncryptionLegacyKeyNonProductionDefaultsToHistoric 验证非生产环境未设置 legacy 时 // 回退到历史硬编码密钥(开发态零配置兼容旧密文)。 func TestFieldEncryptionLegacyKeyNonProductionDefaultsToHistoric(t *testing.T) { diff --git a/backend/internal/modules/chat/handler_qrcode.go b/backend/internal/modules/chat/handler_qrcode.go index a57ac3a7..656aa99a 100644 --- a/backend/internal/modules/chat/handler_qrcode.go +++ b/backend/internal/modules/chat/handler_qrcode.go @@ -103,7 +103,7 @@ func (h *Handler) GetQrCodeStatsHandler(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"data": stats}) } -// RecognizeQrCodeGroupNameHandler 同步调用 PaddleOCR API 识别二维码图片中的企业微信群名(保留向后兼容) +// RecognizeQrCodeGroupNameHandler 同步识别二维码图片中的企业微信群名(保留向后兼容) func (h *Handler) RecognizeQrCodeGroupNameHandler(c *gin.Context) { file, header, err := c.Request.FormFile("file") if err != nil { @@ -120,7 +120,7 @@ func (h *Handler) RecognizeQrCodeGroupNameHandler(c *gin.Context) { ) if err != nil { if errors.Is(err, ErrQrCodeOCRNotConfigured) { - c.JSON(http.StatusBadRequest, gin.H{"error": "请先在系统配置中填写 PaddleOCR API Token"}) + c.JSON(http.StatusBadRequest, gin.H{"error": "二维码 OCR 未配置"}) return } if errors.Is(err, ErrQrCodeOCRInvalidFile) { @@ -129,7 +129,7 @@ func (h *Handler) RecognizeQrCodeGroupNameHandler(c *gin.Context) { } if errors.Is(err, ErrQrCodeOCRUnavailable) { response.RecordError(c, err) - c.JSON(http.StatusBadGateway, gin.H{"error": "PaddleOCR 服务暂时不可用"}) + c.JSON(http.StatusBadGateway, gin.H{"error": "OCR 服务暂时不可用"}) return } response.RecordError(c, err) @@ -140,7 +140,7 @@ func (h *Handler) RecognizeQrCodeGroupNameHandler(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"data": result}) } -// SubmitOCRJobHandler 提交 OCR 任务到 PaddleOCR,立即返回任务 ID +// SubmitOCRJobHandler 提交 OCR 任务,立即返回任务 ID func (h *Handler) SubmitOCRJobHandler(c *gin.Context) { file, header, err := c.Request.FormFile("file") if err != nil { @@ -157,7 +157,7 @@ func (h *Handler) SubmitOCRJobHandler(c *gin.Context) { ) if err != nil { if errors.Is(err, ErrQrCodeOCRNotConfigured) { - c.JSON(http.StatusBadRequest, gin.H{"error": "请先在系统配置中填写 PaddleOCR API Token"}) + c.JSON(http.StatusBadRequest, gin.H{"error": "二维码 OCR 未配置"}) return } if errors.Is(err, ErrQrCodeOCRInvalidFile) { @@ -166,7 +166,7 @@ func (h *Handler) SubmitOCRJobHandler(c *gin.Context) { } if errors.Is(err, ErrQrCodeOCRUnavailable) { response.RecordError(c, err) - c.JSON(http.StatusBadGateway, gin.H{"error": "PaddleOCR 服务暂时不可用"}) + c.JSON(http.StatusBadGateway, gin.H{"error": "OCR 服务暂时不可用"}) return } if errors.Is(err, ErrQrCodeOCRRedisDisabled) { diff --git a/backend/internal/modules/chat/qrcode_ocr.go b/backend/internal/modules/chat/qrcode_ocr.go index d7322e92..316258ce 100644 --- a/backend/internal/modules/chat/qrcode_ocr.go +++ b/backend/internal/modules/chat/qrcode_ocr.go @@ -57,7 +57,7 @@ const ( ) var ( - ErrQrCodeOCRNotConfigured = errors.New("二维码 OCR Token 未配置") + ErrQrCodeOCRNotConfigured = errors.New("二维码 OCR 未配置") ErrQrCodeOCRInvalidFile = errors.New("无效的二维码图片") ErrQrCodeOCRUnavailable = errors.New("二维码 OCR 服务不可用") ErrQrCodeOCRJobNotFound = errors.New("OCR 任务不存在或已过期") @@ -177,6 +177,25 @@ type paddleOCRJobStatusResponse struct { Error string `json:"error"` } +type localOCRLine struct { + Text string `json:"text"` + Score float64 `json:"score"` + X1 float64 `json:"x1"` + Y1 float64 `json:"y1"` + X2 float64 `json:"x2"` + Y2 float64 `json:"y2"` + X3 float64 `json:"x3"` + Y3 float64 `json:"y3"` + X4 float64 `json:"x4"` + Y4 float64 `json:"y4"` +} + +type localOCRResponse struct { + OK bool `json:"ok"` + Result []localOCRLine `json:"result"` + Error string `json:"error"` +} + func readAndValidateOCRFile(reader io.Reader, contentType string) ([]byte, string, error) { data, err := io.ReadAll(io.LimitReader(reader, maxQrCodeOCRFileSize+1)) if err != nil || len(data) == 0 || len(data) > maxQrCodeOCRFileSize { @@ -254,8 +273,16 @@ func (r *Repository) getOCRJobRecord(ctx context.Context, jobID string) (*OCRJob return &record, nil } -// RecognizeQrCodeGroupName 同步调用 PaddleOCR 识别二维码图片中的群名(保留向后兼容)。 +// RecognizeQrCodeGroupName 同步识别二维码图片中的群名(保留向后兼容)。 func (r *Repository) RecognizeQrCodeGroupName(ctx context.Context, filename, contentType string, reader io.Reader) (*QrCodeOCRResult, error) { + data, contentType, err := readAndValidateOCRFile(reader, contentType) + if err != nil { + return nil, err + } + if r.hasOCRService() { + return r.recognizeLocalQrCodeGroupName(ctx, data) + } + config, err := r.GetQrCodeOCRConfig(ctx) if err != nil { return nil, err @@ -263,10 +290,6 @@ func (r *Repository) RecognizeQrCodeGroupName(ctx context.Context, filename, con if strings.TrimSpace(config.Token) == "" { return nil, ErrQrCodeOCRNotConfigured } - data, contentType, err := readAndValidateOCRFile(reader, contentType) - if err != nil { - return nil, err - } paddleJobID, err := r.submitPaddleOCRJob(ctx, config, filename, contentType, data) if err != nil { @@ -292,11 +315,32 @@ func (r *Repository) RecognizeQrCodeGroupName(ctx context.Context, filename, con }, nil } -// SubmitOCRJob 提交 OCR 任务到 PaddleOCR,立即返回任务 ID 供前端轮询。 +// SubmitOCRJob 提交 OCR 任务,立即返回任务 ID 供前端轮询。 func (r *Repository) SubmitOCRJob(ctx context.Context, filename, contentType string, reader io.Reader) (string, error) { if r.redis == nil { return "", ErrQrCodeOCRRedisDisabled } + data, contentType, err := readAndValidateOCRFile(reader, contentType) + if err != nil { + return "", err + } + + jobID := uuid.NewString() + if r.hasOCRService() { + result, err := r.recognizeLocalQrCodeGroupName(ctx, data) + if err != nil { + return "", err + } + record := &OCRJobRecord{ + Status: ocrJobStatusDone, + Result: result, + } + if err := r.saveOCRJobRecord(ctx, jobID, record); err != nil { + return "", err + } + return jobID, nil + } + config, err := r.GetQrCodeOCRConfig(ctx) if err != nil { return "", err @@ -304,17 +348,12 @@ func (r *Repository) SubmitOCRJob(ctx context.Context, filename, contentType str if strings.TrimSpace(config.Token) == "" { return "", ErrQrCodeOCRNotConfigured } - data, contentType, err := readAndValidateOCRFile(reader, contentType) - if err != nil { - return "", err - } paddleJobID, err := r.submitPaddleOCRJob(ctx, config, filename, contentType, data) if err != nil { return "", err } - jobID := uuid.NewString() record := &OCRJobRecord{ PaddleJobID: paddleJobID, Status: ocrJobStatusSubmitted, @@ -325,6 +364,83 @@ func (r *Repository) SubmitOCRJob(ctx context.Context, filename, contentType str return jobID, nil } +func (r *Repository) hasOCRService() bool { + return r != nil && strings.TrimSpace(r.ocrServiceURL) != "" +} + +func (r *Repository) recognizeLocalQrCodeGroupName(ctx context.Context, jpegData []byte) (*QrCodeOCRResult, error) { + ppmData, err := encodeOCRServicePPM(jpegData) + if err != nil { + return nil, ErrQrCodeOCRInvalidFile + } + endpoint := strings.TrimRight(r.ocrServiceURL, "/") + "/api/ocr" + respBody, err := r.doPaddleRequest(ctx, "local", func(ctx context.Context) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(ppmData)) + if err == nil { + req.Header.Set("Content-Type", "image/x-portable-pixmap") + req.Header.Set("Accept", "application/json") + } + return req, err + }) + if err != nil { + return nil, err + } + + var payload localOCRResponse + if err := json.Unmarshal(respBody, &payload); err != nil { + upstreamErr := newOCRUpstreamError("local_decode", http.StatusOK, nil, respBody, err) + r.logOCRUpstreamFailure(ctx, upstreamErr, 1, true, 0) + return nil, upstreamErr + } + if !payload.OK { + message := strings.TrimSpace(payload.Error) + if message == "" { + message = "local OCR service returned ok=false" + } + upstreamErr := newOCRUpstreamError("local_result", http.StatusOK, nil, respBody, errors.New(message)) + r.logOCRUpstreamFailure(ctx, upstreamErr, 1, true, 0) + return nil, upstreamErr + } + + rawText, candidates := parseLocalOCRGroupNames(payload.Result) + groupName := "" + if len(candidates) > 0 { + groupName = candidates[0] + } + return &QrCodeOCRResult{ + GroupName: groupName, + Candidates: candidates, + RawText: rawText, + }, nil +} + +func encodeOCRServicePPM(jpegData []byte) ([]byte, error) { + source, _, err := image.Decode(bytes.NewReader(jpegData)) + if err != nil { + return nil, err + } + bounds := source.Bounds() + width := bounds.Dx() + height := bounds.Dy() + if width <= 0 || height <= 0 { + return nil, ErrQrCodeOCRInvalidFile + } + var buffer bytes.Buffer + buffer.Grow(width*height*3 + 32) + if _, err := fmt.Fprintf(&buffer, "P6\n%d %d\n255\n", width, height); err != nil { + return nil, err + } + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + pixel := color.RGBAModel.Convert(source.At(x, y)).(color.RGBA) + buffer.WriteByte(pixel.R) + buffer.WriteByte(pixel.G) + buffer.WriteByte(pixel.B) + } + } + return buffer.Bytes(), nil +} + // PollOCRJobResult 轮询 OCR 任务状态,每次调用向 PaddleOCR 查询一次并更新缓存。 func (r *Repository) PollOCRJobResult(ctx context.Context, jobID string) (*OCRJobRecord, error) { record, err := r.getOCRJobRecord(ctx, jobID) @@ -654,9 +770,9 @@ func (r *Repository) logOCRUpstreamFailure(ctx context.Context, upstreamErr *ocr fields = append(fields, zap.Error(upstreamErr.Cause)) } if final { - r.ocrLogger(ctx).Error("PaddleOCR 请求失败", fields...) + r.ocrLogger(ctx).Error("OCR 请求失败", fields...) } else { - r.ocrLogger(ctx).Warn("PaddleOCR 请求失败,准备重试", fields...) + r.ocrLogger(ctx).Warn("OCR 请求失败,准备重试", fields...) } } @@ -817,8 +933,8 @@ func collectPaddleOCRText(value any, texts *[]string) { } } -func parseQrCodeGroupNameCandidates(rawText string) []string { - excluded := []*regexp.Regexp{ +var ( + qrCodeGroupNameBoilerplatePatterns = []*regexp.Regexp{ regexp.MustCompile(`使用.*微信`), regexp.MustCompile(`企业微信`), regexp.MustCompile(`扫码`), @@ -828,8 +944,110 @@ func parseQrCodeGroupNameCandidates(rawText string) []string { regexp.MustCompile(`天内`), regexp.MustCompile(`月\d+日`), regexp.MustCompile(`前`), - regexp.MustCompile(`^\d+$`), } + legacyQRCodeGroupNameExcludedPatterns = append( + append([]*regexp.Regexp(nil), qrCodeGroupNameBoilerplatePatterns...), + regexp.MustCompile(`^\d+$`), + ) +) + +func isQrCodeGroupNameBoilerplate(text string) bool { + for _, pattern := range qrCodeGroupNameBoilerplatePatterns { + if pattern.MatchString(text) { + return true + } + } + return false +} + +type localOCRGroupNameCandidate struct { + text string + score float64 + distance float64 + order int +} + +func parseLocalOCRGroupNames(lines []localOCRLine) (string, []string) { + texts := make([]string, 0, len(lines)) + instructionY := math.MaxFloat64 + for _, line := range lines { + text := normalizeQrCodeOCRLine(line.Text) + if text == "" { + continue + } + texts = append(texts, text) + if isQrCodeGroupNameBoilerplate(text) && + (strings.Contains(text, "扫码") || strings.Contains(text, "加入") || strings.Contains(text, "使用")) { + if y := localOCRLineCenterY(line); y < instructionY { + instructionY = y + } + } + } + + candidates := make([]localOCRGroupNameCandidate, 0, len(lines)) + for order, line := range lines { + text := normalizeQrCodeOCRLine(line.Text) + length := utf8.RuneCountInString(text) + if length == 0 || length > 64 || line.Score < 0.45 || isQrCodeGroupNameBoilerplate(text) { + continue + } + y := localOCRLineCenterY(line) + distance := 0.0 + if instructionY < math.MaxFloat64 { + if y >= instructionY-2 { + continue + } + distance = instructionY - y + } + candidates = append(candidates, localOCRGroupNameCandidate{ + text: text, + score: line.Score, + distance: distance, + order: order, + }) + } + sort.SliceStable(candidates, func(i, j int) bool { + if instructionY < math.MaxFloat64 && math.Abs(candidates[i].distance-candidates[j].distance) > 0.5 { + return candidates[i].distance < candidates[j].distance + } + if candidates[i].score != candidates[j].score { + return candidates[i].score > candidates[j].score + } + return candidates[i].order < candidates[j].order + }) + + rawText := strings.Join(texts, "\n") + names := make([]string, 0, len(candidates)+len(lines)) + seen := make(map[string]struct{}, len(candidates)) + add := func(text string) { + if text == "" { + return + } + if _, exists := seen[text]; exists { + return + } + seen[text] = struct{}{} + names = append(names, text) + } + for _, candidate := range candidates { + add(candidate.text) + } + for _, candidate := range parseQrCodeGroupNameCandidates(rawText) { + add(candidate) + } + return rawText, names +} + +func localOCRLineCenterY(line localOCRLine) float64 { + values := []float64{line.Y1, line.Y2, line.Y3, line.Y4} + total := 0.0 + for _, value := range values { + total += value + } + return total / float64(len(values)) +} + +func parseQrCodeGroupNameCandidates(rawText string) []string { scores := map[string]int{} for _, line := range strings.Split(rawText, "\n") { line = normalizeQrCodeOCRLine(line) @@ -840,7 +1058,7 @@ func parseQrCodeGroupNameCandidates(rawText string) []string { continue } skip := false - for _, pattern := range excluded { + for _, pattern := range legacyQRCodeGroupNameExcludedPatterns { if pattern.MatchString(line) { skip = true break diff --git a/backend/internal/modules/chat/qrcode_ocr_test.go b/backend/internal/modules/chat/qrcode_ocr_test.go index 8a46a046..9f817aee 100644 --- a/backend/internal/modules/chat/qrcode_ocr_test.go +++ b/backend/internal/modules/chat/qrcode_ocr_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/binary" + "encoding/json" "errors" "fmt" "hash/crc32" @@ -238,3 +239,77 @@ func TestCollectPaddleOCRJSONLTextSupportsPPOCRv6(t *testing.T) { t.Fatalf("candidates = %v, want 火爆出租-778群 as first candidate", candidates) } } + +func TestParseLocalOCRGroupNamesSupportsShortNumericAndChineseNames(t *testing.T) { + cases := []struct { + name string + title string + }{ + {name: "纯数字", title: "111"}, + {name: "中英数字", title: "小语对接h870"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + lines := []localOCRLine{ + {Text: "C", Score: 0.76, Y1: 73, Y2: 73, Y3: 87, Y4: 87}, + {Text: tc.title, Score: 0.95, Y1: 117, Y2: 117, Y3: 134, Y4: 134}, + {Text: "使用微信或企业微信扫码加入", Score: 0.96, Y1: 163, Y2: 163, Y3: 177, Y4: 177}, + {Text: "该二维码7天内(11月24日前)有效", Score: 0.99, Y1: 487, Y2: 487, Y3: 498, Y4: 498}, + } + rawText, candidates := parseLocalOCRGroupNames(lines) + if len(candidates) == 0 || candidates[0] != tc.title { + t.Fatalf("candidates = %v, want %q; raw=%q", candidates, tc.title, rawText) + } + }) + } +} + +func TestRecognizeQrCodeGroupNameUsesLocalOCRService(t *testing.T) { + var gotContentType string + var gotPPMHeader string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotContentType = r.Header.Get("Content-Type") + body := new(bytes.Buffer) + _, _ = body.ReadFrom(r.Body) + if len(body.Bytes()) >= 2 { + gotPPMHeader = string(body.Bytes()[:2]) + } + _ = json.NewEncoder(w).Encode(localOCRResponse{ + OK: true, + Result: []localOCRLine{ + {Text: "111", Score: 0.999996, Y1: 117, Y2: 117, Y3: 134, Y4: 134}, + {Text: "使用微信或企业微信扫码加入", Score: 0.95, Y1: 163, Y2: 163, Y3: 177, Y4: 177}, + }, + }) + })) + defer server.Close() + + repo := NewRepository(nil, nil, nil, + WithOCRServiceURL(server.URL), + withOCRHTTPClient(server.Client()), + withOCRGate(newOCRRequestGate(1, 0)), + withOCRSleep(func(context.Context, time.Duration) error { return nil }), + ) + imageData := testOCRPNG(t) + result, err := repo.RecognizeQrCodeGroupName(t.Context(), "group.png", "image/png", bytes.NewReader(imageData)) + if err != nil { + t.Fatal(err) + } + if result.GroupName != "111" || !strings.Contains(result.RawText, "使用微信") { + t.Fatalf("result = %+v", result) + } + if gotContentType != "image/x-portable-pixmap" || gotPPMHeader != "P6" { + t.Fatalf("content-type=%q ppm-header=%q", gotContentType, gotPPMHeader) + } +} + +func testOCRPNG(t *testing.T) []byte { + t.Helper() + source := image.NewRGBA(image.Rect(0, 0, 64, 96)) + draw.Draw(source, source.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src) + var buffer bytes.Buffer + if err := png.Encode(&buffer, source); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} diff --git a/backend/internal/modules/chat/repository.go b/backend/internal/modules/chat/repository.go index 86a2133e..76d1b5f4 100644 --- a/backend/internal/modules/chat/repository.go +++ b/backend/internal/modules/chat/repository.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "strings" "time" "github.com/redis/go-redis/v9" @@ -22,6 +23,7 @@ type Repository struct { ocrHTTPClient *http.Client ocrGate *ocrRequestGate ocrSleep func(context.Context, time.Duration) error + ocrServiceURL string } const ( @@ -62,6 +64,12 @@ func withOCRSleep(sleep func(context.Context, time.Duration) error) RepositoryOp } } +func WithOCRServiceURL(rawURL string) RepositoryOption { + return func(repo *Repository) { + repo.ocrServiceURL = strings.TrimRight(strings.TrimSpace(rawURL), "/") + } +} + func NewRepository(db *gorm.DB, hub *chathub.Hub, redis *redis.Client, options ...RepositoryOption) *Repository { repo := &Repository{ db: db, diff --git a/backend/internal/router/router.go b/backend/internal/router/router.go index 30036307..d92587e9 100644 --- a/backend/internal/router/router.go +++ b/backend/internal/router/router.go @@ -177,7 +177,13 @@ func New(cfg config.Config, deps Dependencies, logger *zap.Logger) *gin.Engine { } var chatRepo *chat.Repository if deps.DB != nil { - chatRepo = chat.NewRepository(deps.DB, chatHub, deps.Redis, chat.WithLogger(logger)) + chatRepo = chat.NewRepository( + deps.DB, + chatHub, + deps.Redis, + chat.WithLogger(logger), + chat.WithOCRServiceURL(cfg.OCRServiceURL), + ) } // 创建 chat 适配器用于 listing diff --git a/deploy/README.md b/deploy/README.md index bd3c705a..6911ba5f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -10,7 +10,7 @@ https://选号网域名 # CADDY_SHOW_DOMAIN(可选),hfb_show 静态 https://monitor.主站域名 # Beszel 监控 ``` -生产部署使用 Caddy 作为公网入口,只暴露宿主机 `80`、`443` 端口。Caddy 会自动申请和续签 HTTPS 证书,并直接托管前端静态资源(主站 `dist` 与可选的选号网 `show-dist` 在构建 Caddy 镜像时打入);后端、MySQL、Redis 均通过 Docker 内网通信,图片对象存储使用阿里云 OSS。选号网与主站**共用同一 backend**,浏览器访问选号网域名下的 `/api/*` 由 Caddy 反代到本机 backend,**不会直连数据库**。 +生产部署使用 Caddy 作为公网入口,只暴露宿主机 `80`、`443` 端口。Caddy 会自动申请和续签 HTTPS 证书,并直接托管前端静态资源(主站 `dist` 与可选的选号网 `show-dist` 在构建 Caddy 镜像时打入);后端、MySQL、Redis、OCR sidecar 均通过 Docker 内网通信,图片对象存储使用阿里云 OSS。选号网与主站**共用同一 backend**,浏览器访问选号网域名下的 `/api/*` 由 Caddy 反代到本机 backend,**不会直连数据库**。 部署前请确认: @@ -33,6 +33,7 @@ CADDY_EMAIL TENANT_JIEYOU_DOMAIN OPS_DOMAIN DISTRIBUTION_ADMIN_DOMAIN +OCR_SERVICE_URL MYSQL_ROOT_PASSWORD MYSQL_PASSWORD MYSQL_DSN @@ -223,6 +224,9 @@ docker compose -f deploy/docker-compose.prod.yml run --rm --no-deps backend \ curl https://主站域名/health curl https://主站域名/health/ready curl https://主站域名/api/health +# OCR sidecar 只在 Docker 内网暴露: +docker compose -f deploy/docker-compose.prod.yml exec backend \ + wget -q -O /dev/null http://ocr:8787/health # 若启用了选号网: curl -I https://选号网域名/ curl https://选号网域名/api/health diff --git a/deploy/docker-compose.dev.yml b/deploy/docker-compose.dev.yml index 0848e8c1..bec19e1c 100644 --- a/deploy/docker-compose.dev.yml +++ b/deploy/docker-compose.dev.yml @@ -55,6 +55,16 @@ services: timeout: 5s retries: 10 + ocr: + build: + context: ./ocr + dockerfile: Dockerfile + image: hfb-ocr:dev + container_name: hfb-ocr + restart: unless-stopped + ports: + - "127.0.0.1:${DEV_OCR_PORT:-18787}:8787" + volumes: mysql_data: redis_data: diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 797e2bba..37d67337 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -83,6 +83,24 @@ services: timeout: 5s retries: 10 + ocr: + build: + context: ./ocr + dockerfile: Dockerfile + restart: unless-stopped + environment: + TZ: Asia/Shanghai + deploy: + resources: + limits: + memory: 256M + cpus: '1.0' + reservations: + memory: 128M + cpus: '0.25' + expose: + - "8787" + backend: build: context: ../backend @@ -94,6 +112,7 @@ services: environment: TZ: Asia/Shanghai DISTRIBUTION_ADMIN_DOMAIN: ${DISTRIBUTION_ADMIN_DOMAIN:-distribution.${CADDY_DOMAIN}} + OCR_SERVICE_URL: ${OCR_SERVICE_URL:-http://ocr:8787} volumes: - ../backend/logs:/app/logs # 仅挂载备份状态摘要,后端无法读取备份文件、Docker socket 或 OSS 凭证文件。 @@ -111,6 +130,8 @@ services: condition: service_healthy redis: condition: service_healthy + ocr: + condition: service_healthy expose: - "8080" healthcheck: diff --git a/deploy/ocr/Dockerfile b/deploy/ocr/Dockerfile new file mode 100644 index 00000000..d1d5eaaa --- /dev/null +++ b/deploy/ocr/Dockerfile @@ -0,0 +1,63 @@ +# syntax=docker/dockerfile:1.7 + +FROM ubuntu:24.04 AS build + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + cmake \ + g++ \ + gcc \ + libc6-dev \ + make \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +COPY vendor/lw.PPOCR.C/ ./ + +# Runtime-only avoids Python/model-conversion dependencies. The vendored Tiny +# ONNX models are loaded directly by lw.PPOCR.C. +RUN cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DLW_RUNTIME_ONLY=ON \ + -DLW_BUILD_HTTP_DEMO=OFF \ + -DBUILD_TESTING=OFF \ + && cmake --build build --target lw_ppocr_c --parallel "$(nproc)" \ + && mkdir -p /out \ + && g++ -std=c++11 -O2 \ + -Iinclude \ + -Iexamples/http-server/third_party \ + examples/http-server/main.cpp \ + build/liblw_ppocr_c_static.a \ + -pthread -lm \ + -o /out/lw-http-ocr + +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + libc6 \ + libgcc-s1 \ + libstdc++6 \ + tzdata \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --gid 10001 ocr \ + && useradd --uid 10001 --gid ocr --no-create-home --shell /usr/sbin/nologin ocr + +WORKDIR /app +COPY --from=build /out/lw-http-ocr /usr/local/bin/lw-http-ocr +COPY vendor/lw.PPOCR.C/models/ppocrv6-tiny/ /app/models/ +COPY vendor/lw.PPOCR.C/examples/http-server/www/ /app/www/ + +ENV TZ=Asia/Shanghai +USER 10001:10001 +EXPOSE 8787 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD curl -fsS http://127.0.0.1:8787/health >/dev/null || exit 1 + +ENTRYPOINT ["/usr/local/bin/lw-http-ocr"] +CMD ["--host", "0.0.0.0", "--port", "8787", "--models", "/app/models", "--www", "/app/www", "--ocr-workers", "2", "--rec-max-width", "960"] diff --git a/deploy/ocr/README.md b/deploy/ocr/README.md new file mode 100644 index 00000000..a5dd1907 --- /dev/null +++ b/deploy/ocr/README.md @@ -0,0 +1,23 @@ +# Local OCR sidecar + +This service wraps the vendored `lxw112190/lw.PPOCR.C` runtime and the +PP-OCRv6 Tiny models behind the project's HTTP OCR API. + +It is intentionally private: + +- no host port is published in production; +- only the backend joins the Docker network path to `http://ocr:8787`; +- requests are P6 PPM images; +- the process runs as UID/GID `10001`. + +The Go backend keeps its existing asynchronous OCR job contract and calls this +service only when `OCR_SERVICE_URL` is set. If the service is unavailable, the +admin upload flow still allows manually entering the group name. + +Local verification: + +```bash +docker compose -f deploy/docker-compose.dev.yml up -d --build ocr +curl -fsS http://127.0.0.1:18787/health +docker compose -f deploy/docker-compose.dev.yml logs -f ocr +``` diff --git a/deploy/ocr/vendor/lw.PPOCR.C/CMakeLists.txt b/deploy/ocr/vendor/lw.PPOCR.C/CMakeLists.txt new file mode 100644 index 00000000..4e7e39e6 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/CMakeLists.txt @@ -0,0 +1,3087 @@ +cmake_minimum_required(VERSION 3.20) + +project(lw.PPOCR.C VERSION 1.2.1 LANGUAGES C) + +option(LW_BUILD_HTTP_DEMO + "Build the cross-platform cpp-httplib HTTP OCR Demo" ON) +option(LW_BUILD_CSHARP_DEMOS + "Build the .NET Framework 3.5 WinForms Demo on Windows" OFF) +option(LW_RUNTIME_ONLY + "Build only the dependency-free native runtime for an embedding target" OFF) +option(LW_DEPLOY_ONNX_MODELS + "Deploy official ONNX assets in Web/Node packages (LWM remains supported)" ON) +option(LW_WASM_SIMD128 + "Enable portable WebAssembly SIMD128 for the standalone OCR HTML" ON) +option(LW_WASM_COMPILED_REC + "Experimentally run WASM REC on the compiled physical backend" OFF) +option(LW_WASM_FINE_REC_WIDTHS + "Use 64-pixel adaptive REC width buckets in compiled WASM OCR" OFF) +option(LW_NATIVE_FINE_REC_WIDTHS + "Experimentally reduce native compiled REC padding with finer widths (including 480)" OFF) +option(LW_EXPERIMENTAL_REC_FFN_TILING + "Experimental cache-local Pointwise/GELU/Pointwise scheduling (no arithmetic changes)" OFF) +set(LW_REC_FFN_TILE_PIXELS "96" CACHE STRING "Experimental FFN pixel block (12, 48 or 96)") +set_property(CACHE LW_REC_FFN_TILE_PIXELS PROPERTY STRINGS 12 48 96) +if(NOT LW_REC_FFN_TILE_PIXELS MATCHES "^(12|48|96)$") + message(FATAL_ERROR "LW_REC_FFN_TILE_PIXELS must be 12, 48 or 96") +endif() +option(LW_WASM_COMPILED_DET + "Run WASM DET on the compiled NHWC backend when compiled REC is enabled" ON) +option(LW_WASM_REC_SIMD_KERNELS + "Use SIMD128 Dense and Depthwise kernels in experimental WASM compiled REC" ON) +set(LW_WASM_POINTWISE_ROWS "2" CACHE STRING + "WASM SIMD128 pointwise pixel tile; supported values: 2 or 4") +set_property(CACHE LW_WASM_POINTWISE_ROWS PROPERTY STRINGS 2 4) +if(NOT LW_WASM_POINTWISE_ROWS STREQUAL "2" AND + NOT LW_WASM_POINTWISE_ROWS STREQUAL "4") + message(FATAL_ERROR "LW_WASM_POINTWISE_ROWS must be 2 or 4") +endif() +option(LW_WASM_DB_BITMAP_SIMD + "Experimentally use SIMD128 DB thresholding in compiled WASM OCR" OFF) +option(LW_WEB_LEGACY + "Build the Tiny scalar Web compatibility distribution" OFF) +set(LW_WEB_MIN_CHROME_VERSION "70" CACHE STRING + "Minimum Chromium version for the legacy browser distribution") +option(LW_REC_RESIDENT_WIDTHS + "Prewarm all standard REC widths for native performance experiments" OFF) +option(LW_EXPERIMENTAL_AVX2_FMA_DISPATCH + "Use experimental AVX2+FMA Conv1x1 and terminal MatMul paths in native builds" OFF) +option(LW_EXPERIMENTAL_AVX2_FMA_CONV1X1_DISPATCH + "Use the experimental AVX2+FMA Conv1x1 dispatch in native builds" OFF) +option(LW_EXPERIMENTAL_AVX2_FMA_MATMUL_DISPATCH + "Use the experimental AVX2+FMA terminal MatMul dispatch in native builds" OFF) +option(LW_EXPERIMENTAL_AVX2_FAST_PATH + "Enable compiled AVX2/FMA x64 REC backends for adaptive widths" ON) +option(LW_EXPERIMENTAL_ADAPTIVE_REC_BUDGET + "Share the native x64 REC intra-op thread budget across active line workers" ON) +option(LW_EXPERIMENTAL_DET_FIXED_POINT_RESIZE + "Use the OpenCV-parity fixed-point DET resize with LUT normalization" ON) +option(LW_EXPERIMENTAL_CLS_FIXED_POINT_RESIZE + "Use the keep-aspect fixed-point CLS resize with -1 padding" ON) +option(LW_EXPERIMENTAL_PERSISTENT_LINE_POOL + "Reuse a persistent line-worker thread pool across OCR requests" ON) +option(LW_EXPERIMENTAL_PARALLEL_CROP_PHASE + "Rasterize perspective crops as parallel row-band jobs" ON) +option(LW_EXPERIMENTAL_PREPARED_EXECUTION + "Build the experimental session-local prepared execution table" OFF) +option(LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2 + "Use the experimental size-descending workspace interval planner" OFF) +option(LW_EXPERIMENTAL_FUSION_MEMORY + "Use the experimental physical lifetime plan for fused GELU" OFF) +option(LW_EXPERIMENTAL_CTC_TILED + "Use the experimental physical plan for greedy CTC output elision" OFF) +option(LW_WASM_REC_LAZY_FALLBACK + "Avoid canonical REC execution allocations until a compiled WASM fallback is needed" OFF) +option(LW_EXPERIMENTAL_STREAMING_CROPS + "Use experimental worker-local streaming crop buffers" OFF) +option(LW_AVX2_FMA_CONV3X3_DISPATCH + "Enable the measured AVX2+FMA Conv3x3 Node 6 dispatch" ON) +option(LW_EXPERIMENTAL_AVX2_CONV3X3_EIGHT_OUTPUTS + "Use experimental AVX2 eight-output blocking for unit-stride Conv3x3" OFF) +set(LW_WASM_HOST_ABI_VERSION "1" CACHE STRING + "WASM Host ABI version embedded in the Web adapter and Node manifest") +set(LW_LWM_FORMAT_VERSION "0.1" CACHE STRING + "LWM model format version recorded in distribution manifests") +option(LW_WEB_PDF + "Embed offline PDF.js support in the standalone OCR HTML" ON) +option(LW_BUILD_WEB_MODEL_VARIANTS + "Build standalone PP-OCRv6 Small and Medium browser SDK/HTML variants" OFF) +set(LW_PRECONVERTED_ASSETS_DIR "" CACHE PATH + "Directory containing preconverted LWM models and PPM fixtures") +set(LW_BUILD_ENVIRONMENT_FILE "" CACHE FILEPATH + "Optional build-environment report installed in the package root") +set(LW_STAGED_PACKAGE_TEST_TIMEOUT 240 CACHE STRING + "Timeout in seconds for the installed-package smoke test") +set(LW_HTTP_TEST_REQUEST_TIMEOUT 30 CACHE STRING + "Timeout in seconds for one HTTP package-test request") +set(LW_HTTP_TEST_PROCESS_TIMEOUT 120 CACHE STRING + "Timeout in seconds for the complete HTTP package-test process") +set(LW_PACKAGE_PLATFORM_OVERRIDE "" CACHE STRING + "Optional explicit platform suffix for the CPack archive name") + +if(LW_BUILD_HTTP_DEMO) + enable_language(CXX) +endif() + +if(NOT LW_WASM_HOST_ABI_VERSION MATCHES "^[0-9]+$") + message(FATAL_ERROR "LW_WASM_HOST_ABI_VERSION must be a positive integer") +endif() +if(NOT LW_LWM_FORMAT_VERSION MATCHES "^[0-9]+\\.[0-9]+$") + message(FATAL_ERROR "LW_LWM_FORMAT_VERSION must use the form major.minor") +endif() +if(LW_EXPERIMENTAL_FUSION_MEMORY AND NOT LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2) + message(FATAL_ERROR + "LW_EXPERIMENTAL_FUSION_MEMORY requires LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2") +endif() +if(LW_EXPERIMENTAL_CTC_TILED AND NOT LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2) + message(FATAL_ERROR + "LW_EXPERIMENTAL_CTC_TILED requires LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2") +endif() + +if(LW_WEB_LEGACY) + if(LW_WASM_SIMD128) + message(FATAL_ERROR + "LW_WEB_LEGACY requires LW_WASM_SIMD128=OFF") + endif() + if(LW_WEB_PDF) + message(FATAL_ERROR + "LW_WEB_LEGACY requires LW_WEB_PDF=OFF") + endif() + if(LW_BUILD_WEB_MODEL_VARIANTS) + message(FATAL_ERROR + "LW_WEB_LEGACY supports PP-OCRv6 Tiny only") + endif() + if(NOT LW_WEB_MIN_CHROME_VERSION MATCHES "^[0-9]+$") + message(FATAL_ERROR + "LW_WEB_MIN_CHROME_VERSION must be an integer") + endif() +endif() + +include(CTest) +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) +include(CheckCCompilerFlag) +if(NOT LW_RUNTIME_ONLY) + find_package(Python3 3.9 COMPONENTS Interpreter REQUIRED) +endif() +if(NOT EMSCRIPTEN) + find_package(Threads REQUIRED) +endif() + +if(NOT LW_RUNTIME_ONLY) +set(LW_PPOCR_C_ANALYSIS_JSON + "${CMAKE_CURRENT_SOURCE_DIR}/docs/ppocrv6-tiny-analysis.json") +set(LW_PPOCR_C_ANALYSIS_MARKDOWN + "${CMAKE_CURRENT_SOURCE_DIR}/docs/SUPPORTED_OPS_V0.md") +set(LW_PPOCR_C_REC_ONNX + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/rec.onnx") +set(LW_PPOCR_C_REC_LWM + "${CMAKE_CURRENT_BINARY_DIR}/models/rec.lwm") +set(LW_PPOCR_C_REC_LWM_METADATA + "${CMAKE_CURRENT_BINARY_DIR}/models/rec-lwm.json") +set(LW_PPOCR_C_CLS_ONNX + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/cls.onnx") +set(LW_PPOCR_C_CLS_LWM + "${CMAKE_CURRENT_BINARY_DIR}/models/cls.lwm") +set(LW_PPOCR_C_CLS_LWM_METADATA + "${CMAKE_CURRENT_BINARY_DIR}/models/cls-lwm.json") +set(LW_PPOCR_C_DET_ONNX + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/det.onnx") +set(LW_PPOCR_C_DET_LWM + "${CMAKE_CURRENT_BINARY_DIR}/models/det.lwm") +set(LW_PPOCR_C_DET_LWM_METADATA + "${CMAKE_CURRENT_BINARY_DIR}/models/det-lwm.json") +if(LW_DEPLOY_ONNX_MODELS) + set(LW_DEPLOY_MODEL_FORMAT onnx) + set(LW_DEPLOY_DET "${LW_PPOCR_C_DET_ONNX}") + set(LW_DEPLOY_CLS "${LW_PPOCR_C_CLS_ONNX}") + set(LW_DEPLOY_REC "${LW_PPOCR_C_REC_ONNX}") +else() + set(LW_DEPLOY_MODEL_FORMAT lwm) + set(LW_DEPLOY_DET "${LW_PPOCR_C_DET_LWM}") + set(LW_DEPLOY_CLS "${LW_PPOCR_C_CLS_LWM}") + set(LW_DEPLOY_REC "${LW_PPOCR_C_REC_LWM}") +endif() + +add_custom_target(analyze_models + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/converter/analyze_onnx.py" + --json-output "${LW_PPOCR_C_ANALYSIS_JSON}" + --markdown-output "${LW_PPOCR_C_ANALYSIS_MARKDOWN}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Analyzing bundled PP-OCRv6 tiny ONNX models" + VERBATIM) + +if(LW_PRECONVERTED_ASSETS_DIR) + get_filename_component(LW_PRECONVERTED_ASSETS_DIR + "${LW_PRECONVERTED_ASSETS_DIR}" ABSOLUTE) + foreach(asset IN ITEMS + rec.lwm rec-lwm.json cls.lwm cls-lwm.json det.lwm det-lwm.json + sample-crop.ppm sample.ppm) + if(NOT EXISTS "${LW_PRECONVERTED_ASSETS_DIR}/${asset}") + message(FATAL_ERROR + "LW_PRECONVERTED_ASSETS_DIR is missing required file: ${asset}") + endif() + endforeach() + message(STATUS + "Using preconverted models and fixtures from ${LW_PRECONVERTED_ASSETS_DIR}") + + function(lw_copy_preconverted_pair output_a name_a output_b name_b label) + add_custom_command( + OUTPUT "${output_a}" "${output_b}" + COMMAND "${CMAKE_COMMAND}" -E make_directory + "${CMAKE_CURRENT_BINARY_DIR}/models" + COMMAND "${CMAKE_COMMAND}" -E copy_if_different + "${LW_PRECONVERTED_ASSETS_DIR}/${name_a}" "${output_a}" + COMMAND "${CMAKE_COMMAND}" -E copy_if_different + "${LW_PRECONVERTED_ASSETS_DIR}/${name_b}" "${output_b}" + DEPENDS + "${LW_PRECONVERTED_ASSETS_DIR}/${name_a}" + "${LW_PRECONVERTED_ASSETS_DIR}/${name_b}" + COMMENT "Copying preconverted ${label} assets" + VERBATIM) + endfunction() + + lw_copy_preconverted_pair( + "${LW_PPOCR_C_REC_LWM}" rec.lwm + "${LW_PPOCR_C_REC_LWM_METADATA}" rec-lwm.json REC) + lw_copy_preconverted_pair( + "${LW_PPOCR_C_CLS_LWM}" cls.lwm + "${LW_PPOCR_C_CLS_LWM_METADATA}" cls-lwm.json CLS) + lw_copy_preconverted_pair( + "${LW_PPOCR_C_DET_LWM}" det.lwm + "${LW_PPOCR_C_DET_LWM_METADATA}" det-lwm.json DET) +else() + add_custom_command( + OUTPUT "${LW_PPOCR_C_REC_LWM}" "${LW_PPOCR_C_REC_LWM_METADATA}" + COMMAND "${Python3_EXECUTABLE}" -m converter.convert_rec + --input "${LW_PPOCR_C_REC_ONNX}" + --output "${LW_PPOCR_C_REC_LWM}" + --metadata-output "${LW_PPOCR_C_REC_LWM_METADATA}" + DEPENDS + "${LW_PPOCR_C_REC_ONNX}" + "${CMAKE_CURRENT_SOURCE_DIR}/converter/convert_rec.py" + "${CMAKE_CURRENT_SOURCE_DIR}/converter/lwm_v0.py" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Converting PP-OCRv6 tiny REC to LWM v0.1" + VERBATIM) + + add_custom_command( + OUTPUT "${LW_PPOCR_C_CLS_LWM}" "${LW_PPOCR_C_CLS_LWM_METADATA}" + COMMAND "${Python3_EXECUTABLE}" -m converter.convert_cls + --input "${LW_PPOCR_C_CLS_ONNX}" + --output "${LW_PPOCR_C_CLS_LWM}" + --metadata-output "${LW_PPOCR_C_CLS_LWM_METADATA}" + DEPENDS + "${LW_PPOCR_C_CLS_ONNX}" + "${CMAKE_CURRENT_SOURCE_DIR}/converter/convert_cls.py" + "${CMAKE_CURRENT_SOURCE_DIR}/converter/lwm_v0.py" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Converting PP-OCRv6 tiny CLS to LWM v0.1" + VERBATIM) + + add_custom_command( + OUTPUT "${LW_PPOCR_C_DET_LWM}" "${LW_PPOCR_C_DET_LWM_METADATA}" + COMMAND "${Python3_EXECUTABLE}" -m converter.convert_det + --input "${LW_PPOCR_C_DET_ONNX}" + --output "${LW_PPOCR_C_DET_LWM}" + --metadata-output "${LW_PPOCR_C_DET_LWM_METADATA}" + DEPENDS + "${LW_PPOCR_C_DET_ONNX}" + "${CMAKE_CURRENT_SOURCE_DIR}/converter/convert_det.py" + "${CMAKE_CURRENT_SOURCE_DIR}/converter/lwm_v0.py" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Converting PP-OCRv6 tiny DET to LWM v0.1" + VERBATIM) +endif() + +add_custom_target(convert_rec_model ALL + DEPENDS "${LW_PPOCR_C_REC_LWM}" "${LW_PPOCR_C_REC_LWM_METADATA}") +add_custom_target(convert_cls_model ALL + DEPENDS "${LW_PPOCR_C_CLS_LWM}" "${LW_PPOCR_C_CLS_LWM_METADATA}") +add_custom_target(convert_det_model ALL + DEPENDS "${LW_PPOCR_C_DET_LWM}" "${LW_PPOCR_C_DET_LWM_METADATA}") +endif() + +set(LW_PPOCR_C_SOURCES + src/kernels/packed_conv.c + src/kernels/packed_conv3x3.c + src/kernels/packed_matmul.c + src/kernels/scalar_activation.c + src/kernels/scalar_conv.c + src/kernels/scalar_elementwise.c + src/kernels/scalar_layout.c + src/kernels/scalar_matmul.c + src/kernels/scalar_reduction.c + src/kernels/scalar_softmax.c + src/ppocr/classifier.c + src/ppocr/cls_preprocess.c + src/ppocr/ctc_decode.c + src/ppocr/crop.c + src/ppocr/crop_buffer.c + src/ppocr/db_postprocess.c + src/ppocr/det_preprocess.c + src/ppocr/det_preprocess_fixed.c + src/ppocr/detector.c + src/ppocr/ocr.c + src/ppocr/profile.c + src/ppocr/reading_order.c + src/ppocr/rec_preprocess.c + src/ppocr/recognizer.c + src/runtime/executor.c + src/runtime/cpu_topology.c + src/runtime/execution_plan.c + src/runtime/memory.c + src/runtime/model.c + src/runtime/onnx_import.c + src/runtime/parallel.c + src/runtime/ctc_head_parallel_internal.c + src/runtime/prepared_execution.c + src/runtime/session.c + src/runtime/shape.c + src/runtime/validate.c + src/simd/avx2_binary.c + src/simd/avx2_bitmap.c + src/simd/avx2_conv1x1.c + src/simd/avx2_conv2x2.c + src/simd/avx2_conv3x3.c + src/simd/avx2_conv3x3_eight.c + src/simd/avx2_conv7x7.c + src/simd/avx2_convtranspose2x2.c + src/simd/avx2_depthwise3x3.c + src/simd/avx2_depthwise5x5.c + src/simd/avx2_depthwise9x9.c + src/simd/avx2_erf.c + src/simd/avx2_transpose.c + src/simd/avx2_softmax.c + src/simd/avx2_matmul.c + src/simd/avx2_fma_packed_matmul.c + src/simd/avx2_packed_conv1x1.c + src/simd/avx2_fma_packed_conv1x1.c + src/simd/avx2_fma_packed_conv1x1_8x8.c + src/simd/cpu_features.c + src/simd/lsx_packed_conv1x1.c + src/simd/neon_conv3x3.c + src/simd/neon_convtranspose2x2.c + src/simd/neon_packed_conv1x1.c + src/simd/sse2_binary.c + src/simd/sse2_conv1x1.c + src/simd/sse2_conv2x2.c + src/simd/sse2_conv3x3.c + src/simd/sse2_convtranspose2x2.c + src/simd/sse2_depthwise3x3.c + src/simd/sse2_depthwise5x5.c + src/simd/sse2_matmul.c + src/simd/sse2_packed_conv1x1.c + src/simd/wasm128_erf.c) + +set(LW_X64_REC_FAST_ENABLED OFF) +set(LW_WASM_COMPILED_REC_ENABLED OFF) +if(LW_EXPERIMENTAL_AVX2_FAST_PATH AND NOT EMSCRIPTEN AND + (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64|x64|X86)$" OR + MSVC_C_ARCHITECTURE_ID MATCHES "^(x64|X86)$")) + set(LW_X64_REC_FAST_ENABLED ON) +endif() +if(EMSCRIPTEN AND LW_WASM_SIMD128 AND LW_WASM_COMPILED_REC) + set(LW_WASM_COMPILED_REC_ENABLED ON) +elseif(EMSCRIPTEN AND LW_WASM_COMPILED_REC) + message(FATAL_ERROR "LW_WASM_COMPILED_REC requires LW_WASM_SIMD128=ON") +endif() +if(LW_WASM_REC_LAZY_FALLBACK AND + (NOT LW_WASM_COMPILED_REC_ENABLED OR LW_EXPERIMENTAL_CTC_TILED)) + message(FATAL_ERROR + "LW_WASM_REC_LAZY_FALLBACK requires compiled SIMD128 REC without CTC tiled mode") +endif() +if(LW_WASM_FINE_REC_WIDTHS AND NOT LW_WASM_COMPILED_REC_ENABLED) + message(FATAL_ERROR "LW_WASM_FINE_REC_WIDTHS requires compiled SIMD128 WASM REC") +endif() +if(LW_NATIVE_FINE_REC_WIDTHS AND + (NOT LW_X64_REC_FAST_ENABLED OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8)) + message(FATAL_ERROR "LW_NATIVE_FINE_REC_WIDTHS requires compiled native x64 REC") +endif() +if(LW_X64_REC_FAST_ENABLED OR LW_WASM_COMPILED_REC_ENABLED) + list(APPEND LW_PPOCR_C_SOURCES + src/runtime/layout_planner.c + src/runtime/x64_rec_fast_plan.c + src/runtime/x64_rec_fast_executor.c + src/runtime/x64_rec_backend_compile.c + src/runtime/x64_rec_backend_execute.c + src/runtime/x64_rec_backend_arena.c + src/runtime/rec_backend_kernels.c + src/runtime/x64_det_backend_compile.c + src/runtime/x64_det_backend_execute.c + src/runtime/ctc_projection_internal.c + src/runtime/layout_convert_x64.c + src/kernels/nhwc_pack.c + src/kernels/nhwc_depthwise.c + src/simd/avx2_nhwc_pointwise.c + src/simd/avx2_nhwc_depthwise.c + src/simd/avx2_nhwc_support.c + src/simd/avx2_nhwc_dense.c + src/simd/avx2_nhwc_convtranspose.c) +endif() +if(LW_WASM_COMPILED_REC_ENABLED) + list(APPEND LW_PPOCR_C_SOURCES + src/simd/wasm128_nhwc_pointwise.c + src/simd/wasm128_elementwise.c + src/simd/wasm128_matmul.c + src/simd/wasm128_nhwc_convtranspose.c + src/simd/wasm128_packed_matmul.c) + if(LW_WASM_DB_BITMAP_SIMD) + list(APPEND LW_PPOCR_C_SOURCES src/simd/wasm128_bitmap.c) + endif() +endif() +if(MSVC AND MSVC_C_ARCHITECTURE_ID MATCHES "^(x64|X86)$") + set_property(SOURCE src/simd/avx2_binary.c src/simd/avx2_conv1x1.c + src/simd/avx2_conv2x2.c + src/simd/avx2_conv3x3.c src/simd/avx2_conv3x3_eight.c + src/simd/avx2_conv7x7.c + src/simd/avx2_convtranspose2x2.c + src/simd/avx2_depthwise3x3.c + src/simd/avx2_depthwise5x5.c + src/simd/avx2_depthwise9x9.c + src/simd/avx2_erf.c src/simd/avx2_transpose.c src/simd/avx2_matmul.c + src/simd/avx2_fma_packed_matmul.c + src/simd/avx2_packed_conv1x1.c src/simd/avx2_fma_packed_conv1x1.c + src/simd/avx2_fma_packed_conv1x1_8x8.c APPEND PROPERTY + COMPILE_OPTIONS /arch:AVX2 /fp:strict) + set_property(SOURCE src/simd/cpu_features.c APPEND PROPERTY + COMPILE_OPTIONS /arch:AVX) +endif() + +string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" LW_SYSTEM_PROCESSOR_LOWER) + +# Keep the portable Conv references and their SIMD implementations on the same +# multiply-then-add contract. GCC enables FP contraction by default on AArch64 +# and LoongArch, while the intrinsic kernels below deliberately use separate +# multiply and add instructions so their outputs remain byte-identical to the +# established x86 kernels and golden checksums. +set(LW_REPRODUCIBLE_CONV_SOURCES + src/kernels/packed_conv.c + src/kernels/packed_conv3x3.c + src/kernels/scalar_conv.c + src/simd/lsx_packed_conv1x1.c + src/simd/neon_conv3x3.c + src/simd/neon_convtranspose2x2.c + src/simd/neon_packed_conv1x1.c) + +if(MSVC AND LW_SYSTEM_PROCESSOR_LOWER MATCHES "^(arm64|aarch64)$") + set_property(SOURCE ${LW_REPRODUCIBLE_CONV_SOURCES} APPEND PROPERTY + COMPILE_OPTIONS /fp:strict) +elseif(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + set_property(SOURCE ${LW_REPRODUCIBLE_CONV_SOURCES} APPEND PROPERTY + COMPILE_OPTIONS -ffp-contract=off) +endif() + +if(LW_SYSTEM_PROCESSOR_LOWER MATCHES "^(loongarch64|loong64)$") + check_c_compiler_flag("-mlsx" LW_COMPILER_SUPPORTS_LSX) + if(NOT LW_COMPILER_SUPPORTS_LSX) + message(FATAL_ERROR "The LoongArch compiler must support -mlsx") + endif() + set_property(SOURCE src/simd/lsx_packed_conv1x1.c APPEND PROPERTY + COMPILE_OPTIONS -mlsx -mno-lasx) +endif() + +function(lw_configure_runtime_target target) + if(LW_EXPERIMENTAL_REC_FFN_TILING) + target_compile_definitions(${target} PRIVATE LW_EXPERIMENTAL_REC_FFN_TILING=1 + LW_REC_FFN_TILE_PIXELS=${LW_REC_FFN_TILE_PIXELS}) + endif() + if(LW_NATIVE_FINE_REC_WIDTHS AND LW_X64_REC_FAST_ENABLED) + target_compile_definitions(${target} PRIVATE LW_NATIVE_FINE_REC_WIDTHS=1) + endif() + target_include_directories(${target} + PUBLIC + "$" + "$" + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime" + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(${target} PUBLIC c_std_11) + if(NOT EMSCRIPTEN) + target_link_libraries(${target} PUBLIC Threads::Threads) + endif() + if(MSVC) + target_compile_options(${target} PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(${target} PRIVATE + -Wall -Wextra -Wpedantic -Werror) + target_link_libraries(${target} PUBLIC m) + endif() + if(EMSCRIPTEN AND LW_WASM_SIMD128) + # Keep the offline single-HTML build single-threaded, but compile its + # scalar/SSE2-compatible kernels to portable WebAssembly SIMD128. + # -msse2 enables Emscripten's compatible surface; + # -msimd128 lowers those intrinsics to portable WASM SIMD instructions. + target_compile_options(${target} PRIVATE -O3 -msimd128 -msse2) + endif() +endfunction() + +add_library(lw_ppocr_c STATIC ${LW_PPOCR_C_SOURCES}) +set_target_properties(lw_ppocr_c PROPERTIES + EXPORT_NAME static + OUTPUT_NAME lw_ppocr_c_static + POSITION_INDEPENDENT_CODE ON) +lw_configure_runtime_target(lw_ppocr_c) +if(LW_X64_REC_FAST_ENABLED OR LW_WASM_COMPILED_REC_ENABLED) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + if(LW_X64_REC_FAST_ENABLED AND LW_EXPERIMENTAL_ADAPTIVE_REC_BUDGET) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_ADAPTIVE_REC_BUDGET=1) + endif() + if(LW_WASM_COMPILED_REC_ENABLED) + if(LW_WASM_FINE_REC_WIDTHS) + target_compile_definitions(lw_ppocr_c PRIVATE LW_WASM_FINE_REC_WIDTHS=1) + endif() + target_compile_definitions(lw_ppocr_c PRIVATE + LW_WASM_POINTWISE_ROWS=${LW_WASM_POINTWISE_ROWS}) + endif() + if(LW_WASM_COMPILED_REC_ENABLED AND LW_WASM_REC_SIMD_KERNELS) + target_compile_definitions(lw_ppocr_c PRIVATE LW_WASM_REC_SIMD_KERNELS=1) + endif() + if(LW_WASM_COMPILED_REC_ENABLED AND LW_WASM_DB_BITMAP_SIMD) + target_compile_definitions(lw_ppocr_c PRIVATE LW_WASM_DB_BITMAP_SIMD=1) + endif() + if(LW_WASM_COMPILED_REC_ENABLED AND LW_WASM_COMPILED_DET AND + LW_EXPERIMENTAL_DET_FIXED_POINT_RESIZE) + target_compile_definitions(lw_ppocr_c PRIVATE LW_WASM_COMPILED_DET=1) + endif() + if(MSVC) + set_property(SOURCE src/simd/avx2_nhwc_pointwise.c src/simd/avx2_nhwc_dense.c src/simd/avx2_nhwc_depthwise.c src/simd/avx2_nhwc_support.c src/simd/avx2_nhwc_convtranspose.c APPEND PROPERTY + COMPILE_OPTIONS /arch:AVX2 /fp:strict) + endif() +endif() +if(LW_WASM_REC_LAZY_FALLBACK) + target_compile_definitions(lw_ppocr_c PRIVATE LW_WASM_REC_LAZY_FALLBACK=1) +endif() +if(LW_EXPERIMENTAL_DET_FIXED_POINT_RESIZE) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_DET_FIXED_POINT_RESIZE=1) +endif() +if(LW_EXPERIMENTAL_CLS_FIXED_POINT_RESIZE) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_CLS_FIXED_POINT_RESIZE=1) +endif() +if(LW_EXPERIMENTAL_PERSISTENT_LINE_POOL) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_PERSISTENT_LINE_POOL=1) +endif() +if(LW_EXPERIMENTAL_PARALLEL_CROP_PHASE) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_PARALLEL_CROP_PHASE=1) +endif() +if(LW_REC_RESIDENT_WIDTHS AND NOT EMSCRIPTEN) + target_compile_definitions(lw_ppocr_c PRIVATE LW_REC_RESIDENT_WIDTHS=1) +endif() +if((LW_EXPERIMENTAL_AVX2_FMA_DISPATCH OR LW_EXPERIMENTAL_AVX2_FMA_CONV1X1_DISPATCH) AND NOT EMSCRIPTEN) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_AVX2_FMA_CONV1X1_DISPATCH=1) +endif() +if((LW_EXPERIMENTAL_AVX2_FMA_DISPATCH OR LW_EXPERIMENTAL_AVX2_FMA_MATMUL_DISPATCH) AND NOT EMSCRIPTEN) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_AVX2_FMA_MATMUL_DISPATCH=1) +endif() +if(LW_AVX2_FMA_CONV3X3_DISPATCH AND NOT EMSCRIPTEN) + target_compile_definitions(lw_ppocr_c PRIVATE LW_AVX2_FMA_CONV3X3_DISPATCH=1) +endif() +if(LW_EXPERIMENTAL_AVX2_CONV3X3_EIGHT_OUTPUTS AND NOT EMSCRIPTEN) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_AVX2_CONV3X3_EIGHT_OUTPUTS=1) +endif() +if(LW_EXPERIMENTAL_PREPARED_EXECUTION) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_PREPARED_EXECUTION=1) +endif() +if(LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2=1) +endif() +if(LW_EXPERIMENTAL_FUSION_MEMORY) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_FUSION_MEMORY=1) +endif() +if(LW_EXPERIMENTAL_CTC_TILED) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_CTC_TILED=1) +endif() +if(LW_EXPERIMENTAL_STREAMING_CROPS) + target_compile_definitions(lw_ppocr_c PRIVATE LW_EXPERIMENTAL_STREAMING_CROPS=1) +endif() + +if(NOT EMSCRIPTEN AND NOT LW_RUNTIME_ONLY) +add_library(lw_ppocr_c_shared SHARED ${LW_PPOCR_C_SOURCES}) +set_target_properties(lw_ppocr_c_shared PROPERTIES + EXPORT_NAME shared + OUTPUT_NAME lw_ppocr_c + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR} + C_VISIBILITY_PRESET hidden) +target_compile_definitions(lw_ppocr_c_shared + PRIVATE LW_PPOCR_C_BUILD LW_PPOCR_C_SHARED + INTERFACE LW_PPOCR_C_SHARED) +lw_configure_runtime_target(lw_ppocr_c_shared) +if(LW_X64_REC_FAST_ENABLED) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + if(LW_EXPERIMENTAL_ADAPTIVE_REC_BUDGET) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_ADAPTIVE_REC_BUDGET=1) + endif() + if(MSVC) + set_property(SOURCE src/simd/avx2_nhwc_pointwise.c src/simd/avx2_nhwc_dense.c src/simd/avx2_nhwc_depthwise.c src/simd/avx2_nhwc_support.c src/simd/avx2_nhwc_convtranspose.c APPEND PROPERTY + COMPILE_OPTIONS /arch:AVX2 /fp:strict) + endif() +endif() +if(LW_EXPERIMENTAL_DET_FIXED_POINT_RESIZE) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_DET_FIXED_POINT_RESIZE=1) +endif() +if(LW_EXPERIMENTAL_CLS_FIXED_POINT_RESIZE) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_CLS_FIXED_POINT_RESIZE=1) +endif() +if(LW_EXPERIMENTAL_PERSISTENT_LINE_POOL) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_PERSISTENT_LINE_POOL=1) +endif() +if(LW_EXPERIMENTAL_PARALLEL_CROP_PHASE) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_PARALLEL_CROP_PHASE=1) +endif() +if(LW_REC_RESIDENT_WIDTHS) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_REC_RESIDENT_WIDTHS=1) +endif() +if(LW_EXPERIMENTAL_AVX2_FMA_DISPATCH OR LW_EXPERIMENTAL_AVX2_FMA_CONV1X1_DISPATCH) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_AVX2_FMA_CONV1X1_DISPATCH=1) +endif() +if(LW_EXPERIMENTAL_AVX2_FMA_DISPATCH OR LW_EXPERIMENTAL_AVX2_FMA_MATMUL_DISPATCH) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_AVX2_FMA_MATMUL_DISPATCH=1) +endif() +if(LW_AVX2_FMA_CONV3X3_DISPATCH AND NOT EMSCRIPTEN) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_AVX2_FMA_CONV3X3_DISPATCH=1) +endif() +if(LW_EXPERIMENTAL_AVX2_CONV3X3_EIGHT_OUTPUTS) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_AVX2_CONV3X3_EIGHT_OUTPUTS=1) +endif() +if(LW_EXPERIMENTAL_PREPARED_EXECUTION) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_PREPARED_EXECUTION=1) +endif() +if(LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2=1) +endif() +if(LW_EXPERIMENTAL_FUSION_MEMORY) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_FUSION_MEMORY=1) +endif() +if(LW_EXPERIMENTAL_CTC_TILED) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_CTC_TILED=1) +endif() +if(LW_EXPERIMENTAL_STREAMING_CROPS) + target_compile_definitions(lw_ppocr_c_shared PRIVATE LW_EXPERIMENTAL_STREAMING_CROPS=1) +endif() +endif() + +if(LW_RUNTIME_ONLY) + message(STATUS "LW_RUNTIME_ONLY enabled: skipping model conversion, demos, tests, install and packaging") + return() +endif() + +if(EMSCRIPTEN) + if(LW_WEB_LEGACY) + find_program(LW_NODE_EXECUTABLE NAMES node nodejs REQUIRED) + set(LW_LEGACY_ESBUILD_ENTRY + "${CMAKE_CURRENT_SOURCE_DIR}/web/node_modules/esbuild/package.json") + if(NOT EXISTS "${LW_LEGACY_ESBUILD_ENTRY}") + message(FATAL_ERROR + "LW_WEB_LEGACY requires web/node_modules; run npm ci in web first") + endif() + endif() + add_executable(lw_ppocr_web web/lw_web_api.c) + target_link_libraries(lw_ppocr_web PRIVATE lw_ppocr_c) + target_include_directories(lw_ppocr_web PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include") + target_compile_features(lw_ppocr_web PRIVATE c_std_11) + target_compile_options(lw_ppocr_web PRIVATE -O3 -Wall -Wextra -Wpedantic -Werror) + target_compile_definitions(lw_ppocr_web PRIVATE + LW_WEB_ABI_VERSION=${LW_WASM_HOST_ABI_VERSION}) + if(LW_WASM_SIMD128) + target_compile_options(lw_ppocr_web PRIVATE -msimd128 -msse2) + target_link_options(lw_ppocr_web PRIVATE -msimd128) + endif() + if(LW_WEB_LEGACY) + target_link_options(lw_ppocr_web PRIVATE + "-sMIN_CHROME_VERSION=${LW_WEB_MIN_CHROME_VERSION}" + "-sENVIRONMENT=web,webview,worker") + endif() + target_link_options(lw_ppocr_web PRIVATE + -O3 + "-sMODULARIZE=1" "-sEXPORT_NAME=LwPpocrModule" "-sSINGLE_FILE=1" + "-sFORCE_FILESYSTEM=1" "-sALLOW_MEMORY_GROWTH=1" "-sNO_EXIT_RUNTIME=1" + "-sEXPORTED_RUNTIME_METHODS=['FS','HEAPU8','HEAPU32','HEAPF32']" + "-sEXPORTED_FUNCTIONS=['_lw_web_init','_lw_web_shutdown','_lw_web_info_size','_lw_web_get_info','_lw_web_set_reading_order','_lw_web_run','_lw_web_last_error','_lw_web_malloc','_lw_web_free']" + "--no-entry") + set_target_properties(lw_ppocr_web PROPERTIES OUTPUT_NAME "lw_ppocr_web") + if(NOT LW_DEPLOY_ONNX_MODELS) + add_dependencies(lw_ppocr_web convert_det_model convert_cls_model convert_rec_model) + endif() + + # The generated runtime remains a normal Emscripten module. The browser + # SDK is a separate deterministic artifact layered on top of that module, + # so other applications can consume it without the demo HTML. + if(LW_WEB_LEGACY) + set(LW_WEB_OUTPUT_SUFFIX "-legacy") + set(LW_WEB_BUILD_FLAVOR "legacy") + set(LW_WEB_WASM_SIMD128 "false") + set(LW_WEB_PDF_ENABLED "false") + set(LW_WEB_MIN_CHROME_VERSION_VALUE "${LW_WEB_MIN_CHROME_VERSION}") + set(LW_WEB_JS_TARGET_VALUE "chrome${LW_WEB_MIN_CHROME_VERSION}") + set(LW_WEB_HTML_TITLE + "lw.PPOCR.C · PP-OCRv6 Tiny · 兼容版离线 OCR 工作台") + else() + set(LW_WEB_OUTPUT_SUFFIX "") + set(LW_WEB_BUILD_FLAVOR "modern") + if(LW_WASM_SIMD128) + set(LW_WEB_WASM_SIMD128 "true") + else() + set(LW_WEB_WASM_SIMD128 "false") + endif() + if(LW_WEB_PDF) + set(LW_WEB_PDF_ENABLED "true") + else() + set(LW_WEB_PDF_ENABLED "false") + endif() + set(LW_WEB_MIN_CHROME_VERSION_VALUE "") + set(LW_WEB_JS_TARGET_VALUE "") + set(LW_WEB_HTML_TITLE + "lw.PPOCR.C · PP-OCRv6 Tiny · 离线 OCR 工作台") + endif() + + if(LW_WEB_LEGACY) + set(LW_PPOCR_C_LEGACY_RUNTIME_JS + "${CMAKE_CURRENT_BINARY_DIR}/lw_ppocr_web.legacy.js") + set(LW_PPOCR_C_LEGACY_RUNTIME_REPORT + "${CMAKE_CURRENT_BINARY_DIR}/legacy-runtime-report.json") + add_custom_command( + OUTPUT + "${LW_PPOCR_C_LEGACY_RUNTIME_JS}" + "${LW_PPOCR_C_LEGACY_RUNTIME_REPORT}" + COMMAND "${LW_NODE_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/web/transpile_legacy_runtime.mjs" + --input "$/lw_ppocr_web.js" + --output "${LW_PPOCR_C_LEGACY_RUNTIME_JS}" + --target "chrome${LW_WEB_MIN_CHROME_VERSION}" + --report "${LW_PPOCR_C_LEGACY_RUNTIME_REPORT}" + DEPENDS + lw_ppocr_web + "${CMAKE_CURRENT_SOURCE_DIR}/web/transpile_legacy_runtime.mjs" + "${CMAKE_CURRENT_SOURCE_DIR}/web/package-lock.json" + "${CMAKE_CURRENT_SOURCE_DIR}/web/node_modules/esbuild/package.json" + COMMENT "Lowering Legacy Web runtime for Chrome ${LW_WEB_MIN_CHROME_VERSION}" + VERBATIM) + add_custom_target(lw-legacy-runtime + DEPENDS + "${LW_PPOCR_C_LEGACY_RUNTIME_JS}" + "${LW_PPOCR_C_LEGACY_RUNTIME_REPORT}") + set(LW_PPOCR_C_WEB_RUNTIME_FOR_PACKAGE + "${LW_PPOCR_C_LEGACY_RUNTIME_JS}") + set(LW_PPOCR_C_WEB_RUNTIME_DEPENDS lw-legacy-runtime) + else() + set(LW_PPOCR_C_WEB_RUNTIME_FOR_PACKAGE + "$/lw_ppocr_web.js") + set(LW_PPOCR_C_WEB_RUNTIME_DEPENDS) + endif() + set(LW_PPOCR_C_WEB_SDK + "${CMAKE_CURRENT_BINARY_DIR}/lw-ppocr${LW_WEB_OUTPUT_SUFFIX}.js") + add_custom_command( + OUTPUT "${LW_PPOCR_C_WEB_SDK}" + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/web/package_ocr_sdk.py" + --template "${CMAKE_CURRENT_SOURCE_DIR}/web/lw_ppocr_sdk.template.js" + --runtime "${LW_PPOCR_C_WEB_RUNTIME_FOR_PACKAGE}" + --det "${LW_DEPLOY_DET}" --cls "${LW_DEPLOY_CLS}" + --rec "${LW_DEPLOY_REC}" + --dictionary "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + --model-family "PP-OCRv6" + --model-variant "tiny" + --model-display-name "PP-OCRv6 Tiny" + --version "${PROJECT_VERSION}" + --build-flavor "${LW_WEB_BUILD_FLAVOR}" + --wasm-simd128 "${LW_WEB_WASM_SIMD128}" + --pdf-enabled "${LW_WEB_PDF_ENABLED}" + --min-chrome-version "${LW_WEB_MIN_CHROME_VERSION_VALUE}" + --js-target "${LW_WEB_JS_TARGET_VALUE}" + --output "${LW_PPOCR_C_WEB_SDK}" + DEPENDS + lw_ppocr_web + ${LW_PPOCR_C_WEB_RUNTIME_DEPENDS} + "${CMAKE_CURRENT_SOURCE_DIR}/web/lw_ppocr_sdk.template.js" + "${CMAKE_CURRENT_SOURCE_DIR}/web/package_ocr_sdk.py" + "${LW_DEPLOY_DET}" "${LW_DEPLOY_CLS}" "${LW_DEPLOY_REC}" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + COMMENT "Packaging the browser OCR SDK" + VERBATIM) + add_custom_target(lw-ocr-js DEPENDS "${LW_PPOCR_C_WEB_SDK}") + + # The standalone HTML is now only a thin UI shell plus the SDK. Keep it as + # a separate target so CI and downstream users can download either file. + set(LW_PPOCR_C_WEB_HTML + "${CMAKE_CURRENT_BINARY_DIR}/ocr-demo${LW_WEB_OUTPUT_SUFFIX}.html") + set(LW_PPOCR_C_WEB_HTML_PDF_ARGS --no-pdf) + set(LW_PPOCR_C_WEB_HTML_PDF_DEPENDS) + if(LW_WEB_PDF) + set(LW_PPOCR_C_WEB_HTML_PDF_ARGS + --pdf-adapter "${CMAKE_CURRENT_SOURCE_DIR}/web/pdf/lw_pdf_adapter.js" + --pdfjs-core "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/pdf.min.mjs" + --pdfjs-worker "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/pdf.worker.min.mjs" + --pdfjs-jbig2 "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/jbig2.wasm" + --pdfjs-openjpeg "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/openjpeg.wasm" + --pdfjs-qcms "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/qcms_bg.wasm" + --pdfjs-version "6.3.289") + set(LW_PPOCR_C_WEB_HTML_PDF_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/web/pdf/lw_pdf_adapter.js" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/pdf.min.mjs" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/pdf.worker.min.mjs" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/LICENSE" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/VERSION" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/jbig2.wasm" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/openjpeg.wasm" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/qcms_bg.wasm" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/LICENSE_JBIG2" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/LICENSE_OPENJPEG" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/LICENSE_PDFJS_JBIG2" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/LICENSE_PDFJS_OPENJPEG" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/LICENSE_PDFJS_QCMS" + "${CMAKE_CURRENT_SOURCE_DIR}/web/vendor/pdfjs/LICENSE_QCMS") + endif() + add_custom_command( + OUTPUT "${LW_PPOCR_C_WEB_HTML}" + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/web/package_ocr_html.py" + --template "${CMAKE_CURRENT_SOURCE_DIR}/web/ocr-demo.template.html" + --sdk "${LW_PPOCR_C_WEB_SDK}" + --ui "${CMAKE_CURRENT_SOURCE_DIR}/web/ocr-demo-ui.js" + --sponsor "${CMAKE_CURRENT_SOURCE_DIR}/docs/assets/sponsor.jpg" + --title "${LW_WEB_HTML_TITLE}" + ${LW_PPOCR_C_WEB_HTML_PDF_ARGS} + --output "${LW_PPOCR_C_WEB_HTML}" + DEPENDS + lw-ocr-js + "${LW_PPOCR_C_WEB_SDK}" + "${CMAKE_CURRENT_SOURCE_DIR}/web/ocr-demo.template.html" + "${CMAKE_CURRENT_SOURCE_DIR}/web/ocr-demo-ui.js" + "${CMAKE_CURRENT_SOURCE_DIR}/web/package_ocr_html.py" + "${CMAKE_CURRENT_SOURCE_DIR}/docs/assets/sponsor.jpg" + ${LW_PPOCR_C_WEB_HTML_PDF_DEPENDS} + COMMENT "Packaging the standalone offline OCR HTML demo" + VERBATIM) + add_custom_target(lw-ocr-html DEPENDS "${LW_PPOCR_C_WEB_HTML}") + + # Tiny remains the compatibility/default browser artifact above. Small + # and Medium reuse the same generated WASM runtime and only add their + # model staging plus SDK/HTML wrappers when explicitly requested. This + # keeps the normal browser build fast while providing deterministic, + # independently downloadable model variants for release jobs. + if(LW_BUILD_WEB_MODEL_VARIANTS) + set(LW_WEB_VARIANT_ROOT + "${CMAKE_CURRENT_BINARY_DIR}/runtime-models") + set(LW_WEB_VARIANT_COMMON_DEPS + "${CMAKE_CURRENT_SOURCE_DIR}/tools/prepare_ppocrv6_runtime_variant.py" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-models.json" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-small/det.onnx" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-small/rec.onnx" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-medium/det.onnx" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-medium/rec.onnx" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-shared/PP-OCRv6_small_rec_dict.txt" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/convert_small_det.py" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/convert_small_rec.py" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/convert_medium_det.py" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/convert_medium_rec.py" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/validate_model_catalog.py") + + function(lw_add_web_model_variant variant display_name) + set(_variant_conversion_deps) + if(NOT LW_DEPLOY_ONNX_MODELS) + set(_variant_conversion_deps convert_cls_model) + endif() + set(_variant_root + "${LW_WEB_VARIANT_ROOT}/ppocrv6-${variant}") + set(_variant_stage_stamp + "${_variant_root}/runtime-assets.json") + add_custom_command( + OUTPUT "${_variant_stage_stamp}" + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/prepare_ppocrv6_runtime_variant.py" + --variant "${variant}" + --model-format "${LW_DEPLOY_MODEL_FORMAT}" + --build-dir "${CMAKE_CURRENT_BINARY_DIR}" + --output-dir "${_variant_root}" + DEPENDS ${_variant_conversion_deps} + ${LW_WEB_VARIANT_COMMON_DEPS} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Staging PP-OCRv6 ${display_name} browser model" + VERBATIM) + + set(_variant_sdk + "${CMAKE_CURRENT_BINARY_DIR}/lw-ppocr-v6-${variant}.js") + add_custom_command( + OUTPUT "${_variant_sdk}" + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/web/package_ocr_sdk.py" + --template "${CMAKE_CURRENT_SOURCE_DIR}/web/lw_ppocr_sdk.template.js" + --runtime "$/lw_ppocr_web.js" + --det "${_variant_root}/det.${LW_DEPLOY_MODEL_FORMAT}" + --cls "${_variant_root}/cls.${LW_DEPLOY_MODEL_FORMAT}" + --rec "${_variant_root}/rec.${LW_DEPLOY_MODEL_FORMAT}" + --dictionary "${_variant_root}/ppocr_keys.txt" + --model-family "PP-OCRv6" + --model-variant "${variant}" + --model-display-name "${display_name}" + --version "${PROJECT_VERSION}" + --build-flavor "${LW_WEB_BUILD_FLAVOR}" + --wasm-simd128 "${LW_WEB_WASM_SIMD128}" + --pdf-enabled "${LW_WEB_PDF_ENABLED}" + --min-chrome-version "${LW_WEB_MIN_CHROME_VERSION_VALUE}" + --output "${_variant_sdk}" + DEPENDS lw_ppocr_web + "${_variant_stage_stamp}" + "${CMAKE_CURRENT_SOURCE_DIR}/web/lw_ppocr_sdk.template.js" + "${CMAKE_CURRENT_SOURCE_DIR}/web/package_ocr_sdk.py" + COMMENT "Packaging the PP-OCRv6 ${display_name} browser SDK" + VERBATIM) + add_custom_target(lw-ocr-js-${variant} DEPENDS "${_variant_sdk}") + + set(_variant_html + "${CMAKE_CURRENT_BINARY_DIR}/ocr-demo-${variant}.html") + add_custom_command( + OUTPUT "${_variant_html}" + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/web/package_ocr_html.py" + --template "${CMAKE_CURRENT_SOURCE_DIR}/web/ocr-demo.template.html" + --sdk "${_variant_sdk}" + --ui "${CMAKE_CURRENT_SOURCE_DIR}/web/ocr-demo-ui.js" + --sponsor "${CMAKE_CURRENT_SOURCE_DIR}/docs/assets/sponsor.jpg" + --title "lw.PPOCR.C · ${display_name} · 离线 OCR 工作台" + ${LW_PPOCR_C_WEB_HTML_PDF_ARGS} + --output "${_variant_html}" + DEPENDS lw-ocr-js-${variant} + "${_variant_sdk}" + "${CMAKE_CURRENT_SOURCE_DIR}/web/ocr-demo.template.html" + "${CMAKE_CURRENT_SOURCE_DIR}/web/ocr-demo-ui.js" + "${CMAKE_CURRENT_SOURCE_DIR}/web/package_ocr_html.py" + "${CMAKE_CURRENT_SOURCE_DIR}/docs/assets/sponsor.jpg" + ${LW_PPOCR_C_WEB_HTML_PDF_DEPENDS} + COMMENT "Packaging the PP-OCRv6 ${display_name} offline OCR HTML" + VERBATIM) + add_custom_target(lw-ocr-html-${variant} DEPENDS "${_variant_html}") + endfunction() + + lw_add_web_model_variant(small "PP-OCRv6 Small") + lw_add_web_model_variant(medium "PP-OCRv6 Medium") + add_custom_target(lw-ocr-js-all + DEPENDS lw-ocr-js lw-ocr-js-small lw-ocr-js-medium) + add_custom_target(lw-ocr-html-all + DEPENDS lw-ocr-html lw-ocr-html-small lw-ocr-html-medium) + endif() + + if(NOT LW_WEB_LEGACY) + # Node consumes the same WASM Host ABI as the browser. Keep this target + # separate from the browser SDK so the Node release never needs to parse + # or extract assets from ocr-demo.html. + add_executable(lw_ppocr_node web/lw_web_api.c) + target_link_libraries(lw_ppocr_node PRIVATE lw_ppocr_c) + target_include_directories(lw_ppocr_node PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include") + target_compile_features(lw_ppocr_node PRIVATE c_std_11) + target_compile_options(lw_ppocr_node PRIVATE -O3 -Wall -Wextra -Wpedantic -Werror) + target_compile_definitions(lw_ppocr_node PRIVATE + LW_WEB_ABI_VERSION=${LW_WASM_HOST_ABI_VERSION}) + if(LW_WASM_SIMD128) + target_compile_options(lw_ppocr_node PRIVATE -msimd128 -msse2) + target_link_options(lw_ppocr_node PRIVATE -msimd128) + endif() + if(LW_WASM_COMPILED_REC_ENABLED) + # Only the experimental Node package accepts the C profiler switch. + # The browser SDK and canonical Node package keep their normal env policy. + target_include_directories(lw_ppocr_node PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime" + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr") + target_compile_definitions(lw_ppocr_node PRIVATE LW_NODE_WASM_PROFILE=1) + endif() + target_link_options(lw_ppocr_node PRIVATE + -O3 + "-sMODULARIZE=1" "-sEXPORT_NAME=LwPpocrModule" "-sSINGLE_FILE=1" + "-sENVIRONMENT=node" "-sFORCE_FILESYSTEM=1" "-sALLOW_MEMORY_GROWTH=1" + "-sNO_EXIT_RUNTIME=1" + "-sEXPORTED_RUNTIME_METHODS=['FS','HEAPU8','HEAPU32','HEAPF32']" + "-sEXPORTED_FUNCTIONS=['_lw_web_init','_lw_web_shutdown','_lw_web_info_size','_lw_web_get_info','_lw_web_set_reading_order','_lw_web_run','_lw_web_last_error','_lw_web_malloc','_lw_web_free']" + "--no-entry") + set_target_properties(lw_ppocr_node PROPERTIES OUTPUT_NAME "lw_ppocr_node") + if(NOT LW_DEPLOY_ONNX_MODELS) + add_dependencies(lw_ppocr_node convert_det_model convert_cls_model convert_rec_model) + endif() + + if(LW_WASM_SIMD128) + set(LW_NODE_WASM_BACKEND wasm128) + else() + set(LW_NODE_WASM_BACKEND scalar) + endif() + + set(LW_PPOCR_C_NODE_RUNTIME + "${CMAKE_CURRENT_BINARY_DIR}/runtime.cjs") + add_custom_command( + OUTPUT "${LW_PPOCR_C_NODE_RUNTIME}" + COMMAND "${CMAKE_COMMAND}" -E make_directory + "${CMAKE_CURRENT_BINARY_DIR}/node-wasm" + COMMAND "${CMAKE_COMMAND}" -E copy + "$" "${LW_PPOCR_C_NODE_RUNTIME}" + DEPENDS lw_ppocr_node + COMMENT "Preparing the Node.js WASM runtime" + VERBATIM) + add_custom_target(lw-node-wasm DEPENDS "${LW_PPOCR_C_NODE_RUNTIME}") + + set(LW_PPOCR_C_NODE_PACKAGE_DIR + "${CMAKE_CURRENT_BINARY_DIR}/node-wasm") + set(LW_PPOCR_C_NODE_PACKAGE_ARCHIVE + "${CMAKE_CURRENT_BINARY_DIR}/lw.PPOCR.C-${PROJECT_VERSION}-node-wasm.zip") + set(LW_PPOCR_C_NODE_PACKAGE_CHECKSUM + "${LW_PPOCR_C_NODE_PACKAGE_ARCHIVE}.sha256") + add_custom_command( + OUTPUT "${LW_PPOCR_C_NODE_PACKAGE_ARCHIVE}" + "${LW_PPOCR_C_NODE_PACKAGE_CHECKSUM}" + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/package_node_wasm.py" + --runtime "${LW_PPOCR_C_NODE_RUNTIME}" + --det "${LW_DEPLOY_DET}" + --cls "${LW_DEPLOY_CLS}" + --rec "${LW_DEPLOY_REC}" + --dictionary + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + --license "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" + --model-license + "${CMAKE_CURRENT_SOURCE_DIR}/licenses/PaddleOCR-models-APACHE-2.0.txt" + --notices "${CMAKE_CURRENT_SOURCE_DIR}/THIRD-PARTY-NOTICES.md" + --version "${PROJECT_VERSION}" + --wasm-backend "${LW_NODE_WASM_BACKEND}" + --wasm-host-abi-version "${LW_WASM_HOST_ABI_VERSION}" + --lwm-version "${LW_LWM_FORMAT_VERSION}" + --output-dir "${LW_PPOCR_C_NODE_PACKAGE_DIR}" + --archive "${LW_PPOCR_C_NODE_PACKAGE_ARCHIVE}" + DEPENDS lw-node-wasm "${LW_PPOCR_C_NODE_RUNTIME}" + "${LW_DEPLOY_DET}" "${LW_DEPLOY_CLS}" "${LW_DEPLOY_REC}" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/package_node_wasm.py" + "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" + "${CMAKE_CURRENT_SOURCE_DIR}/licenses/PaddleOCR-models-APACHE-2.0.txt" + "${CMAKE_CURRENT_SOURCE_DIR}/THIRD-PARTY-NOTICES.md" + COMMENT "Packaging the Node.js WASM distribution" + VERBATIM) + add_custom_target(lw-node-wasm-package + DEPENDS "${LW_PPOCR_C_NODE_PACKAGE_ARCHIVE}" + "${LW_PPOCR_C_NODE_PACKAGE_CHECKSUM}") + endif() +endif() + +if(EMSCRIPTEN AND LW_WASM_SIMD128) + add_executable(wasm-erf-contract tests/wasm_erf_contract_driver.c) + target_include_directories(wasm-erf-contract PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd" + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels") + target_compile_features(wasm-erf-contract PRIVATE c_std_11) + target_compile_options(wasm-erf-contract PRIVATE + -O3 -Wall -Wextra -Wpedantic -Werror -msimd128) + target_link_options(wasm-erf-contract PRIVATE -O3 -msimd128 "-sENVIRONMENT=node") +endif() + +if(LW_WASM_COMPILED_REC_ENABLED) + add_executable(wasm-pointwise-contract tests/wasm_pointwise_contract_driver.c) + target_include_directories(wasm-pointwise-contract PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd" + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels") + target_compile_features(wasm-pointwise-contract PRIVATE c_std_11) + target_compile_options(wasm-pointwise-contract PRIVATE + -O3 -Wall -Wextra -Wpedantic -Werror -msimd128) + target_link_libraries(wasm-pointwise-contract PRIVATE lw_ppocr_c) + target_link_options(wasm-pointwise-contract PRIVATE -O3 -msimd128 "-sENVIRONMENT=node") + + # Keep the bitwise generic/prepared kernel contract even though the + # compiled prepared-offset experiment did not meet the E2E threshold. + add_executable(wasm-dense-prepared-contract tests/wasm_dense_prepared_contract_driver.c) + target_include_directories(wasm-dense-prepared-contract PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels") + target_compile_features(wasm-dense-prepared-contract PRIVATE c_std_11) + target_compile_options(wasm-dense-prepared-contract PRIVATE + -O3 -Wall -Wextra -Wpedantic -Werror -msimd128) + target_link_libraries(wasm-dense-prepared-contract PRIVATE lw_ppocr_c) + target_link_options(wasm-dense-prepared-contract PRIVATE -O3 -msimd128 "-sENVIRONMENT=node") +endif() + +if(EMSCRIPTEN AND LW_WASM_REC_LAZY_FALLBACK) + # Private CI executable: exercise a source whose canonical REC session was + # released, a compiled clone, and a clone forced onto canonical fallback. + add_executable(wasm-rec-lazy-contract tests/wasm_rec_lazy_contract_driver.c) + target_link_libraries(wasm-rec-lazy-contract PRIVATE lw_ppocr_c m) + target_include_directories(wasm-rec-lazy-contract PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr") + target_compile_features(wasm-rec-lazy-contract PRIVATE c_std_11) + target_compile_definitions(wasm-rec-lazy-contract PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + if(LW_WASM_FINE_REC_WIDTHS) + target_compile_definitions(wasm-rec-lazy-contract PRIVATE LW_WASM_FINE_REC_WIDTHS=1) + endif() + target_compile_options(wasm-rec-lazy-contract PRIVATE + -O3 -Wall -Wextra -Wpedantic -Werror -msimd128) + target_link_options(wasm-rec-lazy-contract PRIVATE + -O3 -msimd128 "-sENVIRONMENT=node" "-sFORCE_FILESYSTEM=1" + "-sALLOW_MEMORY_GROWTH=1" + "SHELL:--embed-file \"${LW_PPOCR_C_REC_LWM}@/models/rec.lwm\"" + "SHELL:--embed-file \"${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt@/models/ppocr_keys.txt\"") + add_dependencies(wasm-rec-lazy-contract convert_rec_model) +endif() + +set(LW_PPOCR_C_REC_DEMO_PPM + "${CMAKE_CURRENT_BINARY_DIR}/models/sample-crop.ppm") +if(LW_PRECONVERTED_ASSETS_DIR) + add_custom_command( + OUTPUT "${LW_PPOCR_C_REC_DEMO_PPM}" + COMMAND "${CMAKE_COMMAND}" -E make_directory + "${CMAKE_CURRENT_BINARY_DIR}/models" + COMMAND "${CMAKE_COMMAND}" -E copy_if_different + "${LW_PRECONVERTED_ASSETS_DIR}/sample-crop.ppm" + "${LW_PPOCR_C_REC_DEMO_PPM}" + DEPENDS "${LW_PRECONVERTED_ASSETS_DIR}/sample-crop.ppm" + COMMENT "Copying preconverted REC demo fixture" + VERBATIM) +else() + add_custom_command( + OUTPUT "${LW_PPOCR_C_REC_DEMO_PPM}" + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/create_rec_demo_fixture.py" + --input "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg" + --output "${LW_PPOCR_C_REC_DEMO_PPM}" + DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/tools/create_rec_demo_fixture.py" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg" + COMMENT "Creating dependency-free REC demo PPM fixture" + VERBATIM) +endif() +add_custom_target(rec_demo_fixture ALL DEPENDS "${LW_PPOCR_C_REC_DEMO_PPM}") + +set(LW_PPOCR_C_DET_DEMO_PPM + "${CMAKE_CURRENT_BINARY_DIR}/models/sample.ppm") +if(LW_PRECONVERTED_ASSETS_DIR) + add_custom_command( + OUTPUT "${LW_PPOCR_C_DET_DEMO_PPM}" + COMMAND "${CMAKE_COMMAND}" -E make_directory + "${CMAKE_CURRENT_BINARY_DIR}/models" + COMMAND "${CMAKE_COMMAND}" -E copy_if_different + "${LW_PRECONVERTED_ASSETS_DIR}/sample.ppm" + "${LW_PPOCR_C_DET_DEMO_PPM}" + DEPENDS "${LW_PRECONVERTED_ASSETS_DIR}/sample.ppm" + COMMENT "Copying preconverted DET demo fixture" + VERBATIM) +else() + add_custom_command( + OUTPUT "${LW_PPOCR_C_DET_DEMO_PPM}" + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/create_det_demo_fixture.py" + --input "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg" + --output "${LW_PPOCR_C_DET_DEMO_PPM}" + DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/tools/create_det_demo_fixture.py" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg" + COMMENT "Creating dependency-free DET demo PPM fixture" + VERBATIM) +endif() +add_custom_target(det_demo_fixture ALL DEPENDS "${LW_PPOCR_C_DET_DEMO_PPM}") + +if(NOT EMSCRIPTEN) +add_executable(lw-recognize-ppm examples/recognize_ppm.c) +target_link_libraries(lw-recognize-ppm PRIVATE lw_ppocr_c_shared) +target_compile_features(lw-recognize-ppm PRIVATE c_std_11) +add_dependencies(lw-recognize-ppm convert_rec_model rec_demo_fixture) +if(WIN32) + target_link_libraries(lw-recognize-ppm PRIVATE shell32) +elseif(APPLE) + set_target_properties(lw-recognize-ppm PROPERTIES INSTALL_RPATH "@loader_path/../lib") +else() + set_target_properties(lw-recognize-ppm PROPERTIES INSTALL_RPATH "$ORIGIN/../lib") +endif() + +add_executable(lw-rec-benchmark examples/benchmark_ppm.c) +target_sources(lw-rec-benchmark PRIVATE src/simd/cpu_features.c) +target_link_libraries(lw-rec-benchmark PRIVATE lw_ppocr_c_shared) +target_include_directories(lw-rec-benchmark PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") +target_compile_features(lw-rec-benchmark PRIVATE c_std_11) +add_dependencies(lw-rec-benchmark convert_rec_model rec_demo_fixture) +if(WIN32) + target_link_libraries(lw-rec-benchmark PRIVATE shell32 psapi) +elseif(APPLE) + set_target_properties(lw-rec-benchmark PROPERTIES + INSTALL_RPATH "@loader_path/../lib") +else() + set_target_properties(lw-rec-benchmark PROPERTIES + INSTALL_RPATH "$ORIGIN/../lib") +endif() + +add_executable(lw-ocr-benchmark examples/ocr_benchmark_ppm.c) +target_sources(lw-ocr-benchmark PRIVATE src/simd/cpu_features.c) +target_link_libraries(lw-ocr-benchmark PRIVATE lw_ppocr_c_shared) +target_include_directories(lw-ocr-benchmark PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") +target_compile_features(lw-ocr-benchmark PRIVATE c_std_11) +add_dependencies(lw-ocr-benchmark convert_det_model convert_cls_model + convert_rec_model det_demo_fixture) +if(WIN32) + target_link_libraries(lw-ocr-benchmark PRIVATE shell32 psapi) +elseif(APPLE) + set_target_properties(lw-ocr-benchmark PROPERTIES + INSTALL_RPATH "@loader_path/../lib") +else() + set_target_properties(lw-ocr-benchmark PROPERTIES + INSTALL_RPATH "$ORIGIN/../lib") +endif() + +add_executable(lw-classify-ppm examples/classify_ppm.c) +target_link_libraries(lw-classify-ppm PRIVATE lw_ppocr_c_shared) +target_compile_features(lw-classify-ppm PRIVATE c_std_11) +add_dependencies(lw-classify-ppm convert_cls_model rec_demo_fixture) +if(WIN32) + target_link_libraries(lw-classify-ppm PRIVATE shell32) +elseif(APPLE) + set_target_properties(lw-classify-ppm PROPERTIES INSTALL_RPATH "@loader_path/../lib") +else() + set_target_properties(lw-classify-ppm PROPERTIES INSTALL_RPATH "$ORIGIN/../lib") +endif() + +add_executable(lw-detect-ppm examples/detect_ppm.c) +target_link_libraries(lw-detect-ppm PRIVATE lw_ppocr_c_shared) +target_compile_features(lw-detect-ppm PRIVATE c_std_11) +add_dependencies(lw-detect-ppm convert_det_model det_demo_fixture) +if(WIN32) + target_link_libraries(lw-detect-ppm PRIVATE shell32) +elseif(APPLE) + set_target_properties(lw-detect-ppm PROPERTIES INSTALL_RPATH "@loader_path/../lib") +else() + set_target_properties(lw-detect-ppm PROPERTIES INSTALL_RPATH "$ORIGIN/../lib") +endif() + +add_executable(lw-ocr-ppm examples/ocr_ppm.c) +target_link_libraries(lw-ocr-ppm PRIVATE lw_ppocr_c_shared) +target_compile_features(lw-ocr-ppm PRIVATE c_std_11) +add_dependencies(lw-ocr-ppm convert_det_model convert_cls_model + convert_rec_model det_demo_fixture) +if(WIN32) + target_link_libraries(lw-ocr-ppm PRIVATE shell32) +elseif(APPLE) + set_target_properties(lw-ocr-ppm PROPERTIES INSTALL_RPATH "@loader_path/../lib") +else() + set_target_properties(lw-ocr-ppm PROPERTIES INSTALL_RPATH "$ORIGIN/../lib") +endif() +if(MSVC) + target_compile_options(lw-classify-ppm PRIVATE /W4 /WX /utf-8) +else() + target_compile_options(lw-classify-ppm PRIVATE + -Wall -Wextra -Wpedantic -Werror) +endif() +if(MSVC) + target_compile_options(lw-detect-ppm PRIVATE /W4 /WX /utf-8) +else() + target_compile_options(lw-detect-ppm PRIVATE + -Wall -Wextra -Wpedantic -Werror) +endif() +if(MSVC) + target_compile_options(lw-ocr-ppm PRIVATE /W4 /WX /utf-8) +else() + target_compile_options(lw-ocr-ppm PRIVATE + -Wall -Wextra -Wpedantic -Werror) +endif() +if(MSVC) + target_compile_options(lw-rec-benchmark PRIVATE /W4 /WX /utf-8) +else() + target_compile_options(lw-rec-benchmark PRIVATE + -Wall -Wextra -Wpedantic -Werror) +endif() +if(MSVC) + target_compile_options(lw-ocr-benchmark PRIVATE /W4 /WX /utf-8) +else() + target_compile_options(lw-ocr-benchmark PRIVATE + -Wall -Wextra -Wpedantic -Werror) +endif() +if(MSVC) + target_compile_options(lw-recognize-ppm PRIVATE /W4 /WX /utf-8) +else() + target_compile_options(lw-recognize-ppm PRIVATE + -Wall -Wextra -Wpedantic -Werror) +endif() + +add_executable(lwm-inspect tools/lwm_inspect.c) +target_link_libraries(lwm-inspect PRIVATE lw_ppocr_c) +if(WIN32) + target_link_libraries(lwm-inspect PRIVATE shell32) +endif() +target_compile_features(lwm-inspect PRIVATE c_std_11) +if(MSVC) + target_compile_options(lwm-inspect PRIVATE /W4 /WX /utf-8) +else() + target_compile_options(lwm-inspect PRIVATE -Wall -Wextra -Wpedantic -Werror) +endif() + +if(LW_BUILD_HTTP_DEMO) + set(LW_HTTPLIB_HEADER + "${CMAKE_CURRENT_SOURCE_DIR}/examples/http-server/third_party/httplib.h") + file(SHA256 "${LW_HTTPLIB_HEADER}" LW_HTTPLIB_SHA256) + if(NOT LW_HTTPLIB_SHA256 STREQUAL + "1c8b44c8ab0498322c08840dbdfcd5006bc28c5a3b140b0c2cf30777b7232fc6") + message(FATAL_ERROR "vendored cpp-httplib checksum mismatch") + endif() + configure_file( + examples/http-server/www/index.html + "${CMAKE_CURRENT_BINARY_DIR}/www/index.html" + COPYONLY) + add_executable(lw-http-ocr examples/http-server/main.cpp) + set_target_properties(lw-http-ocr PROPERTIES + OUTPUT_NAME "lw.PPOCR.C.HttpServer" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin") + foreach(lw_config Debug Release RelWithDebInfo MinSizeRel) + string(TOUPPER "${lw_config}" lw_config_upper) + set_target_properties(lw-http-ocr PROPERTIES + "RUNTIME_OUTPUT_DIRECTORY_${lw_config_upper}" + "${CMAKE_CURRENT_BINARY_DIR}/bin") + endforeach() + target_include_directories(lw-http-ocr PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/include") + target_include_directories(lw-http-ocr SYSTEM PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/examples/http-server/third_party") + target_link_libraries(lw-http-ocr PRIVATE lw_ppocr_c) + target_compile_features(lw-http-ocr PRIVATE cxx_std_11) + add_dependencies(lw-http-ocr convert_det_model convert_cls_model + convert_rec_model) + if(WIN32) + target_compile_definitions(lw-http-ocr PRIVATE + WINVER=0x0601 _WIN32_WINNT=0x0601 + WIN32_LEAN_AND_MEAN NOMINMAX) + target_link_libraries(lw-http-ocr PRIVATE ws2_32) + elseif(UNIX AND NOT APPLE) + target_link_libraries(lw-http-ocr PRIVATE pthread) + endif() + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + target_compile_definitions(lw-http-ocr PRIVATE + LW_HTTP_THREAD_POOL_COUNT=2) + endif() + if(MSVC) + target_compile_options(lw-http-ocr PRIVATE /W4 /WX /utf-8) + target_link_options(lw-http-ocr PRIVATE + /SUBSYSTEM:CONSOLE,6.01 + $<$:/LARGEADDRESSAWARE>) + else() + target_compile_options(lw-http-ocr PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_custom_command(TARGET lw-http-ocr POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy_if_different + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + "${CMAKE_CURRENT_BINARY_DIR}/models/ppocr_keys.txt" + VERBATIM) +endif() + +if(LW_BUILD_CSHARP_DEMOS) + if(NOT WIN32) + message(FATAL_ERROR "LW_BUILD_CSHARP_DEMOS is supported only on Windows") + endif() + find_program(LW_MSBUILD_EXECUTABLE NAMES MSBuild.exe msbuild REQUIRED) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(LW_MANAGED_PLATFORM x64) + else() + set(LW_MANAGED_PLATFORM x86) + endif() + set(LW_MANAGED_ROOT "${CMAKE_CURRENT_BINARY_DIR}/managed") + set(LW_MANAGED_WINFORMS_DIR "${LW_MANAGED_ROOT}/winforms") + set(LW_MANAGED_MODELS_DIR "${LW_MANAGED_ROOT}/models") + set(LW_MANAGED_WINFORMS_EXE + "${LW_MANAGED_WINFORMS_DIR}/lw.PPOCR.C.WinForms.exe") + add_custom_target(lw-csharp-demos ALL + COMMAND "${CMAKE_COMMAND}" -E make_directory + "${LW_MANAGED_WINFORMS_DIR}" "${LW_MANAGED_MODELS_DIR}" + COMMAND "${LW_MSBUILD_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/examples/csharp-winforms/LwPpocrWinForms.csproj" + -t:Restore -p:RestorePackagesConfig=true + -p:RestoreIgnoreFailedSources=true -p:NuGetAudit=false -v:minimal + COMMAND "${LW_MSBUILD_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/examples/csharp-winforms/LwPpocrWinForms.csproj" + -t:Rebuild -p:Configuration=Release + -p:Platform=${LW_MANAGED_PLATFORM} + "-p:OutputPath=${LW_MANAGED_WINFORMS_DIR}/" -v:minimal + COMMAND "${CMAKE_COMMAND}" -E copy_if_different + "$" + "${LW_MANAGED_WINFORMS_DIR}/" + COMMAND "${CMAKE_COMMAND}" -E copy_if_different + "${LW_PPOCR_C_DET_LWM}" "${LW_PPOCR_C_CLS_LWM}" + "${LW_PPOCR_C_REC_LWM}" + "${LW_PPOCR_C_DET_ONNX}" "${LW_PPOCR_C_CLS_ONNX}" "${LW_PPOCR_C_REC_ONNX}" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/model.json" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg" + "${LW_MANAGED_MODELS_DIR}/" + DEPENDS + lw_ppocr_c_shared + convert_det_model convert_cls_model convert_rec_model + "${CMAKE_CURRENT_SOURCE_DIR}/examples/csharp-common/NativeOcr.cs" + "${CMAKE_CURRENT_SOURCE_DIR}/examples/csharp-common/OcrModels.cs" + "${CMAKE_CURRENT_SOURCE_DIR}/examples/csharp-winforms/MainForm.cs" + "${CMAKE_CURRENT_SOURCE_DIR}/examples/csharp-winforms/OcrResultExporter.cs" + COMMENT "Building .NET Framework 3.5 WinForms OCR Demo" + VERBATIM) +endif() + +endif() + +if(BUILD_TESTING AND NOT EMSCRIPTEN) + + add_executable(wasm-ctc-projection-driver + tests/test_wasm_ctc_projection.c + src/simd/wasm128_packed_matmul.c + src/kernels/packed_matmul.c) + target_include_directories(wasm-ctc-projection-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels") + target_compile_features(wasm-ctc-projection-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(wasm-ctc-projection-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(wasm-ctc-projection-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + target_link_libraries(wasm-ctc-projection-driver PRIVATE m) + endif() + add_test(NAME wasm_ctc_projection COMMAND wasm-ctc-projection-driver) + + add_executable(avx2-ctc-projection-driver tests/test_avx2_ctc_projection.c) + target_link_libraries(avx2-ctc-projection-driver PRIVATE lw_ppocr_c) + target_include_directories(avx2-ctc-projection-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(avx2-ctc-projection-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(avx2-ctc-projection-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(avx2-ctc-projection-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + target_link_libraries(avx2-ctc-projection-driver PRIVATE m) + endif() + add_test(NAME avx2_ctc_projection_tails COMMAND avx2-ctc-projection-driver) + set_tests_properties(avx2_ctc_projection_tails PROPERTIES SKIP_RETURN_CODE 77) + + add_executable(nhwc-depthwise-portable-driver + tests/test_nhwc_depthwise_portable.c + src/simd/avx2_nhwc_depthwise.c) + target_include_directories(nhwc-depthwise-portable-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels") + target_compile_features(nhwc-depthwise-portable-driver PRIVATE c_std_11) + target_compile_definitions(nhwc-depthwise-portable-driver PRIVATE + LW_NHWC_FORCE_PORTABLE=1) + if(MSVC) + target_compile_options(nhwc-depthwise-portable-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(nhwc-depthwise-portable-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME nhwc_depthwise_portable COMMAND nhwc-depthwise-portable-driver) + + foreach(policy IN ITEMS default fine) + add_executable(rec-width-policy-${policy} tests/rec_width_policy_driver.c) + target_include_directories(rec-width-policy-${policy} PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr") + target_compile_features(rec-width-policy-${policy} PRIVATE c_std_11) + if(policy STREQUAL "fine") + target_compile_definitions(rec-width-policy-${policy} PRIVATE + LW_NATIVE_FINE_REC_WIDTHS=1 LW_TEST_FINE_WIDTHS=1) + endif() + if(MSVC) + target_compile_options(rec-width-policy-${policy} PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(rec-width-policy-${policy} PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME rec_width_policy_${policy} COMMAND rec-width-policy-${policy}) + endforeach() + add_test(NAME onnx_e2e_comparison_contract + COMMAND "${Python3_EXECUTABLE}" -m unittest -v tests.test_onnx_e2e_comparison) + set_tests_properties(onnx_e2e_comparison_contract PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" TIMEOUT 30) + if(WIN32 AND MSVC AND CMAKE_SIZEOF_VOID_P EQUAL 8 AND + MSVC_C_ARCHITECTURE_ID MATCHES "^(x64|X86)$") + add_test(NAME csharp_image_limits_contract + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_csharp_image_limits.py" + --native "$") + set_tests_properties(csharp_image_limits_contract PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" TIMEOUT 240) + endif() + add_test(NAME web_release_profile_contract + COMMAND "${Python3_EXECUTABLE}" -m unittest -v tests.test_web_release_profile) + set_tests_properties(web_release_profile_contract PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" TIMEOUT 30) + add_test(NAME winforms_release_version_contract + COMMAND "${Python3_EXECUTABLE}" -m unittest -v tests.test_winforms_release) + set_tests_properties(winforms_release_version_contract PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" TIMEOUT 30) + add_test(NAME native_rec_quality_contract + COMMAND "${Python3_EXECUTABLE}" -m unittest -v tests.test_native_rec_quality) + set_tests_properties(native_rec_quality_contract PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" TIMEOUT 30) + + add_test( + NAME source_final_newline_contract + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_source_newlines.py" + --root "${CMAKE_CURRENT_SOURCE_DIR}") + set_tests_properties(source_final_newline_contract PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 30) + set(LW_RESHAPE_FIXTURE_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-fixtures") + set(LW_RESHAPE_FIXTURES + "${LW_RESHAPE_FIXTURE_DIR}/valid.lwm" + "${LW_RESHAPE_FIXTURE_DIR}/two-unknown.lwm" + "${LW_RESHAPE_FIXTURE_DIR}/non-divisible.lwm" + "${LW_RESHAPE_FIXTURE_DIR}/zero.lwm" + "${LW_RESHAPE_FIXTURE_DIR}/overflow.lwm") + add_custom_command( + OUTPUT ${LW_RESHAPE_FIXTURES} + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tools/create_reshape_test_models.py" + --output-dir "${LW_RESHAPE_FIXTURE_DIR}" + DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/tools/create_reshape_test_models.py" + "${CMAKE_CURRENT_SOURCE_DIR}/converter/lwm_v0.py" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Creating direct dynamic Reshape contract fixtures" + VERBATIM) + add_custom_target(dynamic_reshape_fixtures DEPENDS ${LW_RESHAPE_FIXTURES}) + add_executable(test-dynamic-reshape tests/reshape_dynamic_driver.c) + target_link_libraries(test-dynamic-reshape PRIVATE lw_ppocr_c) + target_compile_features(test-dynamic-reshape PRIVATE c_std_11) + add_dependencies(test-dynamic-reshape dynamic_reshape_fixtures) + if(MSVC) + target_compile_options(test-dynamic-reshape PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(test-dynamic-reshape PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME dynamic_reshape_contract + COMMAND test-dynamic-reshape ${LW_RESHAPE_FIXTURES}) + + add_test( + NAME model_analysis + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_analyze_onnx tests.test_compare_model_analysis + tests.test_cls_identity tests.test_model_contract + tests.test_small_validation_bundle + tests.test_model_catalog + tests.test_model_manifest + tests.test_probe_rec_shape_metadata + tests.test_small_contracts + tests.test_medium_contracts + tests.test_ocr_dataset + tests.test_generate_ocr_dataset + tests.test_prepare_ocr_dataset_ppm + tests.test_build_ocr_width_switch_manifest + tests.test_compare_ocr_dataset_runtime + tests.test_collect_rec_shape_profile + tests.test_collect_det_shape_profile + tests.test_compare_ocr_dataset_reports + tests.test_build_engine_comparison_report + tests.test_layout_plan_snapshot + tests.test_summarize_ocr_dataset_reports + tests.test_check_ocr_dataset_regression + tests.test_experimental_ops tests.test_small_ocr_baseline + tests.test_runtime_model_pack + tests.test_release_assets + tests.test_web_ocr_contracts + tests.test_versioning) + set_tests_properties(model_analysis PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME release_metadata_consistency + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/check_release_readiness.py" + --mode preview + --version "${PROJECT_VERSION}") + set_tests_properties(release_metadata_consistency PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME wasm_host_abi_contract + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_web_abi_contract) + set_tests_properties(wasm_host_abi_contract PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME conv_shape_analysis + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_analyze_conv_shapes) + set_tests_properties(conv_shape_analysis PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME rec_aot_pattern_analysis + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_analyze_rec_aot_patterns) + set_tests_properties(rec_aot_pattern_analysis PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME ocr_dispatch_profile_comparison + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_compare_ocr_dispatch_profiles) + set_tests_properties(ocr_dispatch_profile_comparison PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME rec_runtime_profile_comparison + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_compare_rec_runtime_profiles) + set_tests_properties(rec_runtime_profile_comparison PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME opencv_dnn_comparison_helpers + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_compare_opencv_dnn) + set_tests_properties(opencv_dnn_comparison_helpers PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME conv_bn_fusion + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_conv_bn_fusion) + set_tests_properties(conv_bn_fusion PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME rec_converter + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_convert_rec) + set_tests_properties(rec_converter PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME cls_converter + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_convert_cls) + set_tests_properties(cls_converter PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_test( + NAME det_converter + COMMAND "${Python3_EXECUTABLE}" -m unittest -v + tests.test_convert_det) + set_tests_properties(det_converter PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_executable(test-lwm-loader tests/test_model.c) + target_link_libraries(test-lwm-loader PRIVATE lw_ppocr_c) + target_compile_features(test-lwm-loader PRIVATE c_std_11) + add_dependencies(test-lwm-loader convert_rec_model) + if(MSVC) + target_compile_options(test-lwm-loader PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(test-lwm-loader PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME lwm_loader COMMAND test-lwm-loader "${LW_PPOCR_C_REC_LWM}") + + add_executable(test-crop-buffer tests/test_crop_buffer.c) + target_link_libraries(test-crop-buffer PRIVATE lw_ppocr_c) + target_include_directories(test-crop-buffer PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr") + target_compile_features(test-crop-buffer PRIVATE c_std_11) + if(MSVC) + target_compile_options(test-crop-buffer PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(test-crop-buffer PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME crop_buffer COMMAND test-crop-buffer) + add_executable(test-session-planner tests/test_session.c) + target_link_libraries(test-session-planner PRIVATE lw_ppocr_c) + target_include_directories(test-session-planner PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime") + if(LW_EXPERIMENTAL_PREPARED_EXECUTION) + target_compile_definitions(test-session-planner PRIVATE LW_EXPERIMENTAL_PREPARED_EXECUTION=1) + endif() + target_compile_features(test-session-planner PRIVATE c_std_11) + add_dependencies(test-session-planner convert_rec_model) + if(MSVC) + target_compile_options(test-session-planner PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(test-session-planner PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME session_planner COMMAND test-session-planner "${LW_PPOCR_C_REC_LWM}") + add_executable(layout-plan-driver + tests/layout_plan_driver.c + src/runtime/layout_planner.c) + target_link_libraries(layout-plan-driver PRIVATE lw_ppocr_c) + target_include_directories(layout-plan-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime") + target_compile_features(layout-plan-driver PRIVATE c_std_11) + add_dependencies(layout-plan-driver convert_rec_model convert_det_model) + if(MSVC) + target_compile_options(layout-plan-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(layout-plan-driver PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME layout_plan_analysis + COMMAND layout-plan-driver "${LW_PPOCR_C_REC_LWM}" 48 960) + add_test(NAME layout_plan_direct_input + COMMAND layout-plan-driver "${LW_PPOCR_C_REC_LWM}" 48 960 direct-nhwc) + add_test(NAME layout_plan_det + COMMAND layout-plan-driver "${LW_PPOCR_C_DET_LWM}" 640 640 direct-nhwc) + + add_executable(workspace-report-driver tests/workspace_report_driver.c) + target_link_libraries(workspace-report-driver PRIVATE lw_ppocr_c) + target_include_directories(workspace-report-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime") + target_compile_features(workspace-report-driver PRIVATE c_std_11) + add_dependencies(workspace-report-driver convert_rec_model) + if(MSVC) + target_compile_options(workspace-report-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(workspace-report-driver PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + if(LW_EXPERIMENTAL_CTC_TILED) + target_compile_definitions(workspace-report-driver PRIVATE LW_EXPERIMENTAL_CTC_TILED=1) + endif() + add_test(NAME workspace_memory_report + COMMAND workspace-report-driver "${LW_PPOCR_C_REC_LWM}" 320 1) + add_test(NAME workspace_ctc_memory_report + COMMAND workspace-report-driver "${LW_PPOCR_C_REC_LWM}" 320 1 --ctc-greedy) + + add_executable(test-cpu-topology tests/test_cpu_topology.c) + target_link_libraries(test-cpu-topology PRIVATE lw_ppocr_c) + target_include_directories(test-cpu-topology PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime") + target_compile_features(test-cpu-topology PRIVATE c_std_11) + if(MSVC) + target_compile_options(test-cpu-topology PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(test-cpu-topology PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME cpu_topology_policy COMMAND test-cpu-topology) + + add_executable(test-c-abi tests/test_abi.c) + target_link_libraries(test-c-abi PRIVATE lw_ppocr_c) + target_compile_features(test-c-abi PRIVATE c_std_11) + if(MSVC) + target_compile_options(test-c-abi PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(test-c-abi PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME c_abi_contract COMMAND test-c-abi) + + add_executable(test-c-abi-prefix tests/test_abi_prefix.c) + target_link_libraries(test-c-abi-prefix PRIVATE lw_ppocr_c) + target_compile_features(test-c-abi-prefix PRIVATE c_std_11) + add_dependencies(test-c-abi-prefix convert_det_model convert_cls_model convert_rec_model) + if(MSVC) + target_compile_options(test-c-abi-prefix PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(test-c-abi-prefix PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME c_abi_prefix_compatibility + COMMAND test-c-abi-prefix + "${LW_PPOCR_C_DET_LWM}" + "${LW_PPOCR_C_CLS_LWM}" + "${LW_PPOCR_C_REC_LWM}" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt") + + add_executable(abi-layout-driver tests/abi_layout_driver.c) + target_include_directories(abi-layout-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/include") + target_compile_features(abi-layout-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(abi-layout-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(abi-layout-driver PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME abi_layout_contract + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_abi_layout.py" + --driver "$" + --manifest "${CMAKE_CURRENT_SOURCE_DIR}/abi/c-abi-v1-layout.json") + + add_test( + NAME lwm_layout_contract + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_lwm_layout.py" + --manifest "${CMAKE_CURRENT_SOURCE_DIR}/abi/lwm-v0.1-layout.json" + --model "${LW_PPOCR_C_DET_LWM}" + --model "${LW_PPOCR_C_CLS_LWM}" + --model "${LW_PPOCR_C_REC_LWM}") + set_tests_properties(lwm_layout_contract PROPERTIES + DEPENDS "det_converter;cls_converter;rec_converter") + + add_executable(test-recognizer-api tests/test_recognizer.c) + target_link_libraries(test-recognizer-api PRIVATE lw_ppocr_c) + target_compile_features(test-recognizer-api PRIVATE c_std_11) + add_dependencies(test-recognizer-api convert_rec_model) + if(MSVC) + target_compile_options(test-recognizer-api PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(test-recognizer-api PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME recognizer_api_contract + COMMAND test-recognizer-api + "${LW_PPOCR_C_REC_LWM}" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt") + + add_executable(test-classifier-api tests/test_classifier.c) + target_link_libraries(test-classifier-api PRIVATE lw_ppocr_c) + target_compile_features(test-classifier-api PRIVATE c_std_11) + add_dependencies(test-classifier-api convert_cls_model) + if(MSVC) + target_compile_options(test-classifier-api PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(test-classifier-api PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME classifier_api_contract + COMMAND test-classifier-api "${LW_PPOCR_C_CLS_LWM}") + + add_test( + NAME c_demo_classification + COMMAND lw-classify-ppm + "${LW_PPOCR_C_CLS_LWM}" + "${LW_PPOCR_C_REC_DEMO_PPM}") + + add_test( + NAME c_demo_detection + COMMAND lw-detect-ppm + "${LW_PPOCR_C_DET_LWM}" + "${LW_PPOCR_C_DET_DEMO_PPM}") + set_tests_properties(c_demo_detection PROPERTIES TIMEOUT 300) + + add_test( + NAME c_demo_full_ocr + COMMAND lw-ocr-ppm + "${LW_PPOCR_C_DET_LWM}" + "${LW_PPOCR_C_CLS_LWM}" + "${LW_PPOCR_C_REC_LWM}" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + "${LW_PPOCR_C_DET_DEMO_PPM}") + set_tests_properties(c_demo_full_ocr PROPERTIES TIMEOUT 600) + + add_test( + NAME c_demo_recognition + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_recognize_ppm_demo.py" + --demo "$" + --model "${LW_PPOCR_C_REC_LWM}" + --dictionary "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + --image "${LW_PPOCR_C_REC_DEMO_PPM}") + set_tests_properties(c_demo_recognition PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 180) + + add_test( + NAME rec_scalar_benchmark + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_rec_benchmark.py" + --benchmark "$" + --model "${LW_PPOCR_C_REC_LWM}" + --dictionary "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + --image "${LW_PPOCR_C_REC_DEMO_PPM}") + set_tests_properties(rec_scalar_benchmark PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 240) + + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(LW_FULL_OCR_BENCH_WORKERS 8) + else() + set(LW_FULL_OCR_BENCH_WORKERS 1) + endif() + add_test( + NAME full_ocr_parallel_benchmark + COMMAND lw-ocr-benchmark + "${LW_PPOCR_C_DET_LWM}" + "${LW_PPOCR_C_CLS_LWM}" + "${LW_PPOCR_C_REC_LWM}" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + "${LW_PPOCR_C_DET_DEMO_PPM}" + 1 1 "${LW_FULL_OCR_BENCH_WORKERS}" 960) + set_tests_properties(full_ocr_parallel_benchmark PROPERTIES TIMEOUT 300) + + if(MSVC) + get_filename_component(LW_EXPORT_TOOL_DIR "${CMAKE_C_COMPILER}" DIRECTORY) + set(LW_EXPORT_TOOL "${LW_EXPORT_TOOL_DIR}/dumpbin.exe") + else() + set(LW_EXPORT_TOOL "${CMAKE_NM}") + endif() + add_test( + NAME shared_export_contract + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_shared_exports.py" + --library "$" + --allowlist "${CMAKE_CURRENT_SOURCE_DIR}/abi/exports-v0.txt" + --tool "${LW_EXPORT_TOOL}") + set_tests_properties(shared_export_contract PROPERTIES TIMEOUT 60) + + add_test( + NAME shared_export_candidate_contract + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_abi_v1_candidate.py" + --header "${CMAKE_CURRENT_SOURCE_DIR}/include/lw_infer.h" + --manifest "${CMAKE_CURRENT_SOURCE_DIR}/abi/c-abi-v1-candidate.json" + --allowlist "${CMAKE_CURRENT_SOURCE_DIR}/abi/exports-v1-candidate.txt" + --library "$" + --tool "${LW_EXPORT_TOOL}") + set_tests_properties(shared_export_candidate_contract PROPERTIES TIMEOUT 60) + + add_executable(kernel-reference-driver tests/kernel_reference_driver.c) + target_link_libraries(kernel-reference-driver PRIVATE lw_ppocr_c) + target_include_directories(kernel-reference-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(kernel-reference-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(kernel-reference-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(kernel-reference-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME scalar_kernel_reference + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_scalar_kernels_reference.py" + --driver "$") + set_tests_properties(scalar_kernel_reference PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_executable(tensor-reference-driver tests/tensor_reference_driver.c) + target_link_libraries(tensor-reference-driver PRIVATE lw_ppocr_c) + target_include_directories(tensor-reference-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(tensor-reference-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(tensor-reference-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(tensor-reference-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME tensor_kernel_reference + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_tensor_kernels_reference.py" + --driver "$") + set_tests_properties(tensor_kernel_reference PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_executable(conv-reference-driver tests/conv_reference_driver.c) + target_link_libraries(conv-reference-driver PRIVATE lw_ppocr_c) + target_include_directories(conv-reference-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(conv-reference-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(conv-reference-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(conv-reference-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME conv_kernel_reference + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_conv_kernels_reference.py" + --driver "$") + + add_executable(conv3x3-benchmark-driver + tests/conv3x3_benchmark_driver.c) + target_link_libraries(conv3x3-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(conv3x3-benchmark-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(conv3x3-benchmark-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(conv3x3-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(conv3x3-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME conv3x3_benchmark_smoke + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_conv3x3_benchmark.py" + --driver "$") + set_tests_properties(conv3x3_benchmark_smoke PROPERTIES TIMEOUT 120) + + add_executable(convtranspose-benchmark-driver + tests/convtranspose_benchmark_driver.c) + target_link_libraries(convtranspose-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(convtranspose-benchmark-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(convtranspose-benchmark-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(convtranspose-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(convtranspose-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME convtranspose_benchmark_smoke + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_convtranspose_benchmark.py" + --driver "$") + set_tests_properties(convtranspose_benchmark_smoke PROPERTIES TIMEOUT 120) + + add_executable(packed-conv1x1-benchmark-driver + tests/packed_conv1x1_benchmark_driver.c) + target_link_libraries(packed-conv1x1-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(packed-conv1x1-benchmark-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(packed-conv1x1-benchmark-driver PRIVATE c_std_11) + if(LW_EXPERIMENTAL_AVX2_FMA_DISPATCH OR LW_EXPERIMENTAL_AVX2_FMA_CONV1X1_DISPATCH) + target_compile_definitions(packed-conv1x1-benchmark-driver PRIVATE LW_EXPERIMENTAL_AVX2_FMA_CONV1X1_DISPATCH=1) + endif() + if(MSVC) + target_compile_options(packed-conv1x1-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(packed-conv1x1-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME packed_conv1x1_benchmark_smoke + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_packed_conv1x1_benchmark.py" + --driver "$") + set_tests_properties(packed_conv1x1_benchmark_smoke PROPERTIES TIMEOUT 120) + + add_executable(packed-matmul-benchmark-driver + tests/packed_matmul_benchmark_driver.c) + target_link_libraries(packed-matmul-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(packed-matmul-benchmark-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(packed-matmul-benchmark-driver PRIVATE c_std_11) + if(LW_EXPERIMENTAL_AVX2_FMA_DISPATCH OR LW_EXPERIMENTAL_AVX2_FMA_MATMUL_DISPATCH) + target_compile_definitions(packed-matmul-benchmark-driver PRIVATE LW_EXPERIMENTAL_AVX2_FMA_MATMUL_DISPATCH=1) + endif() + if(MSVC) + target_compile_options(packed-matmul-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(packed-matmul-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME packed_matmul_benchmark_smoke + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_packed_matmul_benchmark.py" + --driver "$") + set_tests_properties(packed_matmul_benchmark_smoke PROPERTIES TIMEOUT 120) + add_executable(erf-benchmark-driver + tests/erf_benchmark_driver.c) + target_link_libraries(erf-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(erf-benchmark-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(erf-benchmark-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(erf-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(erf-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME erf_benchmark_smoke + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_erf_benchmark.py" + --driver "$") + set_tests_properties(erf_benchmark_smoke PROPERTIES TIMEOUT 120) + + add_executable(conv3x3-stride2-benchmark-driver + tests/conv3x3_stride2_benchmark_driver.c) + target_link_libraries(conv3x3-stride2-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(conv3x3-stride2-benchmark-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(conv3x3-stride2-benchmark-driver PRIVATE c_std_11) + if(LW_EXPERIMENTAL_AVX2_FMA_DISPATCH) + target_compile_definitions(conv3x3-stride2-benchmark-driver PRIVATE LW_EXPERIMENTAL_AVX2_FMA_DISPATCH=1) + endif() + if(LW_AVX2_FMA_CONV3X3_DISPATCH AND NOT EMSCRIPTEN) + target_compile_definitions(conv3x3-stride2-benchmark-driver PRIVATE LW_AVX2_FMA_CONV3X3_DISPATCH=1) + endif() + if(MSVC) + target_compile_options(conv3x3-stride2-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(conv3x3-stride2-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME conv3x3_stride2_benchmark_smoke + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_conv3x3_stride2_benchmark.py" + --driver "$") + set_tests_properties(conv3x3_stride2_benchmark_smoke PROPERTIES TIMEOUT 120) + set_tests_properties(conv_kernel_reference PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + add_executable(simd-backend-driver tests/simd_backend_driver.c) + target_link_libraries(simd-backend-driver PRIVATE lw_ppocr_c) + target_include_directories(simd-backend-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(simd-backend-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(simd-backend-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(simd-backend-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME simd_backend_detection COMMAND simd-backend-driver) + + add_executable(rec-graph-driver tests/rec_graph_driver.c) + target_link_libraries(rec-graph-driver PRIVATE lw_ppocr_c) + target_include_directories(rec-graph-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime") + target_compile_features(rec-graph-driver PRIVATE c_std_11) + add_dependencies(rec-graph-driver convert_rec_model) + if(MSVC) + target_compile_definitions(rec-graph-driver PRIVATE _CRT_SECURE_NO_WARNINGS) + target_compile_options(rec-graph-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(rec-graph-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME rec_graph_reference + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_rec_graph_reference.py" + --driver "$" + --lwm-model "${LW_PPOCR_C_REC_LWM}" + --onnx-model "${LW_PPOCR_C_REC_ONNX}") + set_tests_properties(rec_graph_reference PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 360) + set(LW_ONNX_WIDTH_TEST_ARGS) + if(LW_NATIVE_FINE_REC_WIDTHS) + list(APPEND LW_ONNX_WIDTH_TEST_ARGS --fine-rec-widths) + endif() + add_test(NAME native_onnx_import + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_onnx_import.py" + --inspect "$" + --rec-driver "$" + --ocr-driver "$" + --root "${CMAKE_CURRENT_SOURCE_DIR}" + --sample "${LW_PPOCR_C_DET_DEMO_PPM}" + ${LW_ONNX_WIDTH_TEST_ARGS} + --report "${CMAKE_CURRENT_BINARY_DIR}/onnx-import-report.json") + set_tests_properties(native_onnx_import PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" TIMEOUT 600) + add_test(NAME deployment_model_formats + COMMAND "${Python3_EXECUTABLE}" -m unittest -v tests.test_deployment_model_formats) + set_tests_properties(deployment_model_formats PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" TIMEOUT 120) + + add_executable(cls-graph-driver tests/cls_graph_driver.c) + target_link_libraries(cls-graph-driver PRIVATE lw_ppocr_c) + target_include_directories(cls-graph-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime") + target_compile_features(cls-graph-driver PRIVATE c_std_11) + add_dependencies(cls-graph-driver convert_cls_model) + if(MSVC) + target_compile_options(cls-graph-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(cls-graph-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME cls_graph_reference + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_cls_graph_reference.py" + --driver "$" + --lwm-model "${LW_PPOCR_C_CLS_LWM}" + --onnx-model "${LW_PPOCR_C_CLS_ONNX}") + set_tests_properties(cls_graph_reference PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 360) + + add_executable(det-graph-driver tests/det_graph_driver.c) + target_link_libraries(det-graph-driver PRIVATE lw_ppocr_c) + target_include_directories(det-graph-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime") + target_compile_features(det-graph-driver PRIVATE c_std_11) + add_dependencies(det-graph-driver convert_det_model) + if(MSVC) + target_compile_definitions(det-graph-driver PRIVATE _CRT_SECURE_NO_WARNINGS) + target_compile_options(det-graph-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(det-graph-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME det_graph_reference + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_det_graph_reference.py" + --driver "$" + --lwm-model "${LW_PPOCR_C_DET_LWM}" + --onnx-model "${LW_PPOCR_C_DET_ONNX}") + set_tests_properties(det_graph_reference PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 600) + + add_executable(det-pipeline-driver tests/det_pipeline_driver.c) + target_link_libraries(det-pipeline-driver PRIVATE lw_ppocr_c) + target_include_directories(det-pipeline-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr") + target_compile_features(det-pipeline-driver PRIVATE c_std_11) + add_dependencies(det-pipeline-driver convert_det_model) + if(MSVC) + target_compile_options(det-pipeline-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(det-pipeline-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME det_pipeline_reference + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_det_pipeline_reference.py" + --driver "$" + --model "${LW_PPOCR_C_DET_LWM}" + --sample "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg") + set_tests_properties(det_pipeline_reference PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 900) + + add_executable(reading-order-driver tests/reading_order_driver.c) + target_link_libraries(reading-order-driver PRIVATE lw_ppocr_c) + target_include_directories(reading-order-driver PRIVATE src/ppocr) + target_compile_features(reading-order-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(reading-order-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(reading-order-driver PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME reading_order_sorting COMMAND reading-order-driver) + + add_executable(ocr-pipeline-driver tests/ocr_pipeline_driver.c) + target_link_libraries(ocr-pipeline-driver PRIVATE lw_ppocr_c) + target_include_directories(ocr-pipeline-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr") + target_compile_features(ocr-pipeline-driver PRIVATE c_std_11) + add_dependencies(ocr-pipeline-driver convert_det_model convert_cls_model + convert_rec_model) + if(MSVC) + target_compile_definitions(ocr-pipeline-driver PRIVATE _CRT_SECURE_NO_WARNINGS) + target_compile_options(ocr-pipeline-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(ocr-pipeline-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME full_ocr_pipeline_reference + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_ocr_pipeline_reference.py" + --driver "$" + --detector "${LW_PPOCR_C_DET_LWM}" + --classifier "${LW_PPOCR_C_CLS_LWM}" + --recognizer "${LW_PPOCR_C_REC_LWM}" + --dictionary "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + --sample "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg") + set_tests_properties(full_ocr_pipeline_reference PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 1200) + + add_test( + NAME full_ocr_golden_corpus + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_ocr_golden_corpus.py" + --driver "$" + --detector "${LW_PPOCR_C_DET_LWM}" + --classifier "${LW_PPOCR_C_CLS_LWM}" + --recognizer "${LW_PPOCR_C_REC_LWM}" + --dictionary "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + --detector-source "${LW_PPOCR_C_DET_ONNX}" + --classifier-source "${LW_PPOCR_C_CLS_ONNX}" + --recognizer-source "${LW_PPOCR_C_REC_ONNX}" + --sample "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg" + --corpus "${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures/ocr-golden-corpus.json") + set_tests_properties(full_ocr_golden_corpus PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 1200) + + add_executable(cls-pipeline-driver tests/cls_pipeline_driver.c) + target_link_libraries(cls-pipeline-driver PRIVATE lw_ppocr_c) + if(WIN32) + target_link_libraries(cls-pipeline-driver PRIVATE shell32) + endif() + target_include_directories(cls-pipeline-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr") + target_compile_features(cls-pipeline-driver PRIVATE c_std_11) + add_dependencies(cls-pipeline-driver convert_cls_model) + if(MSVC) + target_compile_options(cls-pipeline-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(cls-pipeline-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME cls_pipeline_reference + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_cls_pipeline_reference.py" + --driver "$" + --lwm-model "${LW_PPOCR_C_CLS_LWM}" + --onnx-model "${LW_PPOCR_C_CLS_ONNX}") + set_tests_properties(cls_pipeline_reference PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 360) + + add_executable(rec-profile-driver tests/rec_profile_driver.c) + target_link_libraries(rec-profile-driver PRIVATE lw_ppocr_c) + target_include_directories(rec-profile-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime") + target_compile_features(rec-profile-driver PRIVATE c_std_11) + add_dependencies(rec-profile-driver convert_rec_model) + if(MSVC) + target_compile_options(rec-profile-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(rec-profile-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME rec_operator_profile + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_rec_profile.py" + --driver "$" + --model "${LW_PPOCR_C_REC_LWM}") + set_tests_properties(rec_operator_profile PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 240) + + add_executable(full-ocr-profile-driver tests/full_ocr_profile_driver.c) + target_link_libraries(full-ocr-profile-driver PRIVATE lw_ppocr_c) + target_include_directories(full-ocr-profile-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/examples" + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime" + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr") + target_compile_features(full-ocr-profile-driver PRIVATE c_std_11) + add_dependencies(full-ocr-profile-driver convert_det_model convert_cls_model + convert_rec_model det_demo_fixture) + if(MSVC) + target_compile_options(full-ocr-profile-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(full-ocr-profile-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + set(full_ocr_profile_test_args + --driver "$" + --det-model "${LW_PPOCR_C_DET_LWM}" + --cls-model "${LW_PPOCR_C_CLS_LWM}" + --rec-model "${LW_PPOCR_C_REC_LWM}" + --dictionary "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + --image "${LW_PPOCR_C_DET_DEMO_PPM}") + if(LW_REC_RESIDENT_WIDTHS AND NOT EMSCRIPTEN) + list(APPEND full_ocr_profile_test_args --expect-resident) + endif() + if(LW_X64_REC_FAST_ENABLED) + list(APPEND full_ocr_profile_test_args --expect-compiled-rec) + endif() + if(LW_EXPERIMENTAL_PREPARED_EXECUTION AND NOT EMSCRIPTEN) + list(APPEND full_ocr_profile_test_args --expect-prepared) + endif() + add_test( + NAME full_ocr_operator_profile + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_full_ocr_profile.py" + ${full_ocr_profile_test_args}) + set_tests_properties(full_ocr_operator_profile PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 360) + + # Test-only benchmark variant: it links the static runtime so an internal + # DET intra-op override can be measured without expanding the public ABI. + add_executable(full-ocr-intra-benchmark examples/ocr_benchmark_ppm.c) + target_link_libraries(full-ocr-intra-benchmark PRIVATE lw_ppocr_c) + target_compile_definitions(full-ocr-intra-benchmark PRIVATE + LW_OCR_BENCHMARK_INTERNAL=1) + target_include_directories(full-ocr-intra-benchmark PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/examples" + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr" + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(full-ocr-intra-benchmark PRIVATE c_std_11) + add_dependencies(full-ocr-intra-benchmark convert_det_model convert_cls_model + convert_rec_model det_demo_fixture) + if(WIN32) + target_link_libraries(full-ocr-intra-benchmark PRIVATE shell32 psapi) + endif() + if(MSVC) + target_compile_options(full-ocr-intra-benchmark PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(full-ocr-intra-benchmark PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + + # Same-process dataset benchmark used to measure resident/compact memory + # and latency without restarting the OCR runtime for every image. + if(NOT EMSCRIPTEN) + add_executable(full-ocr-dataset-benchmark examples/ocr_dataset_benchmark.c) + target_link_libraries(full-ocr-dataset-benchmark PRIVATE lw_ppocr_c) + target_include_directories(full-ocr-dataset-benchmark PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/examples" + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(full-ocr-dataset-benchmark PRIVATE c_std_11) + add_dependencies(full-ocr-dataset-benchmark convert_det_model convert_cls_model + convert_rec_model det_demo_fixture) + if(LW_REC_RESIDENT_WIDTHS) + target_compile_definitions(full-ocr-dataset-benchmark PRIVATE LW_REC_RESIDENT_WIDTHS=1) + endif() + if(WIN32) + target_link_libraries(full-ocr-dataset-benchmark PRIVATE shell32 psapi) + endif() + if(MSVC) + target_compile_definitions(full-ocr-dataset-benchmark PRIVATE _CRT_SECURE_NO_WARNINGS) + target_compile_options(full-ocr-dataset-benchmark PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(full-ocr-dataset-benchmark PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + if(LW_X64_REC_FAST_ENABLED AND LW_EXPERIMENTAL_ADAPTIVE_REC_BUDGET) + add_test( + NAME full_ocr_adaptive_rec_budget + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_ocr_adaptive_rec_budget.py" + --benchmark "$" + --det "${LW_PPOCR_C_DET_LWM}" + --cls "${LW_PPOCR_C_CLS_LWM}" + --rec "${LW_PPOCR_C_REC_LWM}" + --dictionary "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + --sample "${LW_PPOCR_C_DET_DEMO_PPM}") + set_tests_properties(full_ocr_adaptive_rec_budget PROPERTIES TIMEOUT 180) + endif() + endif() + + add_executable(rec-preprocess-layout-driver tests/rec_preprocess_layout_driver.c) + target_link_libraries(rec-preprocess-layout-driver PRIVATE lw_ppocr_c) + target_include_directories(rec-preprocess-layout-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr") + target_compile_features(rec-preprocess-layout-driver PRIVATE c_std_11) + if(MSVC) + target_compile_definitions(rec-preprocess-layout-driver PRIVATE _CRT_SECURE_NO_WARNINGS) + target_compile_options(rec-preprocess-layout-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(rec-preprocess-layout-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME rec_preprocess_layout COMMAND rec-preprocess-layout-driver) + set_tests_properties(rec_preprocess_layout PROPERTIES TIMEOUT 30) + add_executable(nhwc-conv1x1-driver tests/nhwc_conv1x1_driver.c) + target_link_libraries(nhwc-conv1x1-driver PRIVATE lw_ppocr_c) + target_include_directories(nhwc-conv1x1-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(nhwc-conv1x1-driver PRIVATE c_std_11) + if(MSVC AND CMAKE_SYSTEM_PROCESSOR MATCHES "(AMD64|x86_64|amd64|X86)") + target_compile_options(nhwc-conv1x1-driver PRIVATE /arch:AVX2) + endif() + if(MSVC) + target_compile_definitions(nhwc-conv1x1-driver PRIVATE _CRT_SECURE_NO_WARNINGS) + target_compile_options(nhwc-conv1x1-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(nhwc-conv1x1-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME nhwc_conv1x1_layout COMMAND nhwc-conv1x1-driver) + set_tests_properties(nhwc_conv1x1_layout PROPERTIES TIMEOUT 30) + if(LW_EXPERIMENTAL_AVX2_FAST_PATH AND + (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64|x64|X86)$" OR + MSVC_C_ARCHITECTURE_ID MATCHES "^(x64|X86)$")) + add_executable(nhwc-pointwise-benchmark-driver + tests/nhwc_pointwise_driver.c + src/kernels/nhwc_pack.c + src/simd/avx2_nhwc_pointwise.c) + target_link_libraries(nhwc-pointwise-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(nhwc-pointwise-benchmark-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(nhwc-pointwise-benchmark-driver PRIVATE c_std_11) + target_compile_definitions(nhwc-pointwise-benchmark-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + if(MSVC) + set_property(SOURCE src/simd/avx2_nhwc_pointwise.c APPEND PROPERTY + COMPILE_OPTIONS /arch:AVX2 /fp:strict) + target_compile_definitions(nhwc-pointwise-benchmark-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(nhwc-pointwise-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(nhwc-pointwise-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME nhwc_pointwise_benchmark + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_nhwc_pointwise_benchmark.py" + --driver "$") + set_tests_properties(nhwc_pointwise_benchmark PROPERTIES TIMEOUT 120) + add_executable(nhwc-support-driver + tests/nhwc_support_driver.c) + target_link_libraries(nhwc-support-driver PRIVATE lw_ppocr_c) + target_include_directories(nhwc-support-driver PRIVATE + src/kernels) + target_compile_features(nhwc-support-driver PRIVATE c_std_11) + target_compile_definitions(nhwc-support-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + if(MSVC) + target_compile_definitions(nhwc-support-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(nhwc-support-driver PRIVATE /arch:AVX2 /W4 /WX /utf-8) + else() + target_compile_options(nhwc-support-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME nhwc_support_pool COMMAND nhwc-support-driver) + set_tests_properties(nhwc_support_pool PROPERTIES TIMEOUT 30) + add_executable(nhwc-dense-benchmark-driver + tests/nhwc_dense_driver.c + src/kernels/nhwc_pack.c + src/simd/avx2_nhwc_dense.c) + target_link_libraries(nhwc-dense-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(nhwc-dense-benchmark-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd") + target_compile_features(nhwc-dense-benchmark-driver PRIVATE c_std_11) + target_compile_definitions(nhwc-dense-benchmark-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + if(MSVC) + set_property(SOURCE src/simd/avx2_nhwc_dense.c APPEND PROPERTY + COMPILE_OPTIONS /arch:AVX2 /fp:strict) + target_compile_definitions(nhwc-dense-benchmark-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(nhwc-dense-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(nhwc-dense-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME nhwc_dense_benchmark + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_nhwc_dense_benchmark.py" + --driver "$") + set_tests_properties(nhwc_dense_benchmark PROPERTIES TIMEOUT 120) + add_executable(nhwc-convtranspose-driver + tests/nhwc_convtranspose_driver.c) + target_link_libraries(nhwc-convtranspose-driver PRIVATE lw_ppocr_c) + target_include_directories(nhwc-convtranspose-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels" + "${CMAKE_CURRENT_SOURCE_DIR}/src/simd" + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime") + target_compile_features(nhwc-convtranspose-driver PRIVATE c_std_11) + target_compile_definitions(nhwc-convtranspose-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + if(MSVC) + target_compile_definitions(nhwc-convtranspose-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(nhwc-convtranspose-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(nhwc-convtranspose-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME nhwc_convtranspose_contract COMMAND nhwc-convtranspose-driver) + set_tests_properties(nhwc_convtranspose_contract PROPERTIES TIMEOUT 120) + add_executable(x64-rec-backend-contract-driver + tests/x64_rec_backend_contract_driver.c) + target_link_libraries(x64-rec-backend-contract-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-rec-backend-contract-driver PRIVATE + src/runtime + src/kernels + src/simd) + target_compile_features(x64-rec-backend-contract-driver PRIVATE c_std_11) + target_compile_definitions(x64-rec-backend-contract-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(x64-rec-backend-contract-driver convert_rec_model) + if(MSVC) + target_compile_definitions(x64-rec-backend-contract-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(x64-rec-backend-contract-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-rec-backend-contract-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_rec_backend_contract + COMMAND x64-rec-backend-contract-driver ${LW_PPOCR_C_REC_LWM}) + set_tests_properties(x64_rec_backend_contract PROPERTIES TIMEOUT 120) + add_executable(x64-rec-backend-execution-driver + tests/x64_rec_backend_execution_driver.c) + if(LW_EXPERIMENTAL_REC_FFN_TILING) + add_executable(rec-ffn-contract-driver tests/rec_ffn_contract_driver.c) + target_link_libraries(rec-ffn-contract-driver PRIVATE lw_ppocr_c) + target_include_directories(rec-ffn-contract-driver PRIVATE src/runtime src/kernels src/simd) + target_compile_features(rec-ffn-contract-driver PRIVATE c_std_11) + if(MSVC) + target_compile_options(rec-ffn-contract-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(rec-ffn-contract-driver PRIVATE -Wall -Wextra -Wpedantic -Werror) + endif() + foreach(variant tiny small medium) + add_test(NAME rec_ffn_contract_${variant} + COMMAND rec-ffn-contract-driver + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-${variant}/rec.onnx") + set_tests_properties(rec_ffn_contract_${variant} PROPERTIES TIMEOUT 180) + endforeach() + endif() + target_link_libraries(x64-rec-backend-execution-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-rec-backend-execution-driver PRIVATE + src/runtime + src/kernels + src/simd + src/ppocr) + target_compile_features(x64-rec-backend-execution-driver PRIVATE c_std_11) + target_compile_definitions(x64-rec-backend-execution-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(x64-rec-backend-execution-driver convert_rec_model) + if(MSVC) + target_compile_definitions(x64-rec-backend-execution-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(x64-rec-backend-execution-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-rec-backend-execution-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_rec_backend_execution + COMMAND x64-rec-backend-execution-driver ${LW_PPOCR_C_REC_LWM}) + set_tests_properties(x64_rec_backend_execution PROPERTIES TIMEOUT 120) + add_executable(x64-rec-shared-constants-driver + tests/x64_rec_shared_constants_driver.c) + target_link_libraries(x64-rec-shared-constants-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-rec-shared-constants-driver PRIVATE + src/runtime src/kernels src/simd) + target_compile_features(x64-rec-shared-constants-driver PRIVATE c_std_11) + add_dependencies(x64-rec-shared-constants-driver convert_rec_model) + if(MSVC) + target_compile_options(x64-rec-shared-constants-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-rec-shared-constants-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_rec_shared_constants + COMMAND x64-rec-shared-constants-driver ${LW_PPOCR_C_REC_LWM}) + set_tests_properties(x64_rec_shared_constants PROPERTIES TIMEOUT 120) + add_executable(x64-rec-backend-nchw-driver + tests/x64_rec_backend_nchw_driver.c) + target_link_libraries(x64-rec-backend-nchw-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-rec-backend-nchw-driver PRIVATE + src/runtime + src/kernels + src/simd + src/ppocr) + target_compile_features(x64-rec-backend-nchw-driver PRIVATE c_std_11) + target_compile_definitions(x64-rec-backend-nchw-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(x64-rec-backend-nchw-driver convert_rec_model) + if(MSVC) + target_compile_definitions(x64-rec-backend-nchw-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(x64-rec-backend-nchw-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-rec-backend-nchw-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_rec_backend_nchw + COMMAND x64-rec-backend-nchw-driver ${LW_PPOCR_C_REC_LWM}) + set_tests_properties(x64_rec_backend_nchw PROPERTIES TIMEOUT 120) + add_executable(x64-rec-layout-benchmark-driver + tests/x64_rec_layout_benchmark_driver.c) + target_link_libraries(x64-rec-layout-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-rec-layout-benchmark-driver PRIVATE + src/runtime + src/kernels + src/simd + src/ppocr) + target_compile_features(x64-rec-layout-benchmark-driver PRIVATE c_std_11) + target_compile_definitions(x64-rec-layout-benchmark-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(x64-rec-layout-benchmark-driver convert_rec_model) + if(MSVC) + target_compile_definitions(x64-rec-layout-benchmark-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(x64-rec-layout-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-rec-layout-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_rec_layout_benchmark + COMMAND x64-rec-layout-benchmark-driver ${LW_PPOCR_C_REC_LWM} 5) + set_tests_properties(x64_rec_layout_benchmark PROPERTIES TIMEOUT 120) + foreach(rec_width IN ITEMS 192 320 480 640) + add_test(NAME x64_rec_layout_benchmark_${rec_width} + COMMAND x64-rec-layout-benchmark-driver ${LW_PPOCR_C_REC_LWM} + 5 ${rec_width}) + set_tests_properties(x64_rec_layout_benchmark_${rec_width} + PROPERTIES TIMEOUT 120) + endforeach() + add_executable(x64-det-backend-contract-driver + tests/x64_det_backend_contract_driver.c) + target_link_libraries(x64-det-backend-contract-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-det-backend-contract-driver PRIVATE + src/runtime + src/kernels + src/simd + src/ppocr) + target_compile_features(x64-det-backend-contract-driver PRIVATE c_std_11) + target_compile_definitions(x64-det-backend-contract-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(x64-det-backend-contract-driver convert_det_model) + if(MSVC) + target_compile_definitions(x64-det-backend-contract-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(x64-det-backend-contract-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-det-backend-contract-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_det_backend_contract + COMMAND x64-det-backend-contract-driver ${LW_PPOCR_C_DET_LWM} 32 64) + set_tests_properties(x64_det_backend_contract PROPERTIES TIMEOUT 120) + add_test(NAME x64_det_backend_contract_sharded + COMMAND x64-det-backend-contract-driver ${LW_PPOCR_C_DET_LWM} 640 640) + set_tests_properties(x64_det_backend_contract_sharded PROPERTIES TIMEOUT 180) + add_executable(x64-det-layout-benchmark-driver + tests/x64_det_layout_benchmark_driver.c) + target_link_libraries(x64-det-layout-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-det-layout-benchmark-driver PRIVATE + src/runtime + src/kernels + src/simd + src/ppocr) + target_compile_features(x64-det-layout-benchmark-driver PRIVATE c_std_11) + target_compile_definitions(x64-det-layout-benchmark-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(x64-det-layout-benchmark-driver convert_det_model) + if(MSVC) + target_compile_definitions(x64-det-layout-benchmark-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(x64-det-layout-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-det-layout-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_det_layout_benchmark + COMMAND x64-det-layout-benchmark-driver ${LW_PPOCR_C_DET_LWM} 32 64 1 3) + set_tests_properties(x64_det_layout_benchmark PROPERTIES TIMEOUT 120) + add_executable(x64-rec-backend-benchmark-driver + tests/x64_rec_backend_benchmark_driver.c) + target_link_libraries(x64-rec-backend-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-rec-backend-benchmark-driver PRIVATE + src/runtime + src/kernels + src/simd + src/ppocr) + target_compile_features(x64-rec-backend-benchmark-driver PRIVATE c_std_11) + target_compile_definitions(x64-rec-backend-benchmark-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(x64-rec-backend-benchmark-driver convert_rec_model) + if(MSVC) + target_compile_definitions(x64-rec-backend-benchmark-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(x64-rec-backend-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-rec-backend-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_rec_backend_benchmark + COMMAND x64-rec-backend-benchmark-driver ${LW_PPOCR_C_REC_LWM} 5) + set_tests_properties(x64_rec_backend_benchmark PROPERTIES TIMEOUT 120) + add_test(NAME x64_rec_backend_benchmark_nchw + COMMAND x64-rec-backend-benchmark-driver ${LW_PPOCR_C_REC_LWM} 5 nchw) + set_tests_properties(x64_rec_backend_benchmark_nchw PROPERTIES TIMEOUT 120) + add_executable(x64-rec-recognizer-benchmark-driver + tests/x64_rec_recognizer_benchmark_driver.c) + target_link_libraries(x64-rec-recognizer-benchmark-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-rec-recognizer-benchmark-driver PRIVATE + src/ppocr) + target_compile_features(x64-rec-recognizer-benchmark-driver PRIVATE c_std_11) + target_compile_definitions(x64-rec-recognizer-benchmark-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(x64-rec-recognizer-benchmark-driver convert_rec_model) + if(MSVC) + target_compile_definitions(x64-rec-recognizer-benchmark-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(x64-rec-recognizer-benchmark-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-rec-recognizer-benchmark-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_rec_recognizer_benchmark + COMMAND x64-rec-recognizer-benchmark-driver ${LW_PPOCR_C_REC_LWM} + ${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt 5) + foreach(rec_width IN ITEMS 192 320 480 640) + add_test(NAME x64_rec_recognizer_benchmark_${rec_width} + COMMAND x64-rec-recognizer-benchmark-driver ${LW_PPOCR_C_REC_LWM} + ${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt + 5 ${rec_width}) + set_tests_properties(x64_rec_recognizer_benchmark_${rec_width} + PROPERTIES TIMEOUT 120) + endforeach() + set_tests_properties(x64_rec_recognizer_benchmark PROPERTIES TIMEOUT 120) + add_executable(x64-rec-pointwise-layout-driver + tests/x64_rec_pointwise_layout_driver.c) + target_link_libraries(x64-rec-pointwise-layout-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-rec-pointwise-layout-driver PRIVATE + src/runtime + src/kernels + src/simd) + target_compile_features(x64-rec-pointwise-layout-driver PRIVATE c_std_11) + target_compile_definitions(x64-rec-pointwise-layout-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(x64-rec-pointwise-layout-driver convert_rec_model) + if(MSVC) + target_compile_definitions(x64-rec-pointwise-layout-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(x64-rec-pointwise-layout-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-rec-pointwise-layout-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_rec_pointwise_layout + COMMAND x64-rec-pointwise-layout-driver ${LW_PPOCR_C_REC_LWM} 5) + set_tests_properties(x64_rec_pointwise_layout PROPERTIES TIMEOUT 120) + add_executable(x64-rec-pointwise-tuning-driver + tests/x64_rec_pointwise_tuning_driver.c) + target_link_libraries(x64-rec-pointwise-tuning-driver PRIVATE lw_ppocr_c) + target_include_directories(x64-rec-pointwise-tuning-driver PRIVATE + src/runtime + src/kernels + src/simd + src/ppocr) + target_compile_features(x64-rec-pointwise-tuning-driver PRIVATE c_std_11) + target_compile_definitions(x64-rec-pointwise-tuning-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(x64-rec-pointwise-tuning-driver convert_rec_model) + if(MSVC) + target_compile_definitions(x64-rec-pointwise-tuning-driver PRIVATE + _CRT_SECURE_NO_WARNINGS) + target_compile_options(x64-rec-pointwise-tuning-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(x64-rec-pointwise-tuning-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_executable(rec-fast-prototype-driver tests/rec_fast_prototype_driver.c) + target_link_libraries(rec-fast-prototype-driver PRIVATE lw_ppocr_c) + target_include_directories(rec-fast-prototype-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime" + "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels") + target_compile_features(rec-fast-prototype-driver PRIVATE c_std_11) + target_compile_definitions(rec-fast-prototype-driver PRIVATE + LW_EXPERIMENTAL_AVX2_FAST_PATH=1) + add_dependencies(rec-fast-prototype-driver convert_rec_model) + if(MSVC) + target_compile_definitions(rec-fast-prototype-driver PRIVATE _CRT_SECURE_NO_WARNINGS) + target_compile_options(rec-fast-prototype-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(rec-fast-prototype-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test(NAME x64_rec_fast_prototype + COMMAND rec-fast-prototype-driver "${LW_PPOCR_C_REC_LWM}" 960) + set_tests_properties(x64_rec_fast_prototype PROPERTIES TIMEOUT 120) + elseif(LW_EXPERIMENTAL_AVX2_FAST_PATH) + message(WARNING + "LW_EXPERIMENTAL_AVX2_FAST_PATH requires an x86/x64 build; benchmark target skipped") + endif() + add_executable(rec-pipeline-driver tests/rec_pipeline_driver.c) + target_link_libraries(rec-pipeline-driver PRIVATE lw_ppocr_c) + if(WIN32) + target_link_libraries(rec-pipeline-driver PRIVATE shell32) + endif() + target_include_directories(rec-pipeline-driver PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime" + "${CMAKE_CURRENT_SOURCE_DIR}/src/ppocr") + target_compile_features(rec-pipeline-driver PRIVATE c_std_11) + add_dependencies(rec-pipeline-driver convert_rec_model) + if(MSVC) + target_compile_definitions(rec-pipeline-driver PRIVATE _CRT_SECURE_NO_WARNINGS) + target_compile_options(rec-pipeline-driver PRIVATE /W4 /WX /utf-8) + else() + target_compile_options(rec-pipeline-driver PRIVATE + -Wall -Wextra -Wpedantic -Werror) + endif() + add_test( + NAME rec_pipeline_reference + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_rec_pipeline_reference.py" + --driver "$" + --lwm-model "${LW_PPOCR_C_REC_LWM}" + --onnx-model "${LW_PPOCR_C_REC_ONNX}" + --dictionary "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + --sample "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg") + set_tests_properties(rec_pipeline_reference PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 360) + + add_test( + NAME rec_golden_corpus + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_rec_golden_corpus.py" + --driver "$" + --lwm-model "${LW_PPOCR_C_REC_LWM}" + --onnx-model "${LW_PPOCR_C_REC_ONNX}" + --dictionary "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + --sample "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg" + --corpus "${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures/rec-golden-corpus.json") + set_tests_properties(rec_golden_corpus PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 600) + + add_test( + NAME lwm_corruption + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_lwm_corruption.py" + --inspect "$" + --model "${LW_PPOCR_C_REC_LWM}") + set_tests_properties(lwm_corruption PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + + if(LW_BUILD_HTTP_DEMO) + add_test( + NAME http_demo_smoke + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_http_demo.py" + --server "$" + --models "${CMAKE_CURRENT_BINARY_DIR}/models" + --www "${CMAKE_CURRENT_BINARY_DIR}/www" + --sample "${LW_PPOCR_C_DET_DEMO_PPM}") + set_tests_properties(http_demo_smoke PROPERTIES TIMEOUT 120) + endif() +endif() + +if(MSVC) + set(CMAKE_INSTALL_SYSTEM_RUNTIME_DESTINATION "${CMAKE_INSTALL_BINDIR}") + set(CMAKE_INSTALL_UCRT_LIBRARIES TRUE) + include(InstallRequiredSystemLibraries) +endif() + +if(NOT EMSCRIPTEN) +install(TARGETS lw_ppocr_c lw_ppocr_c_shared + EXPORT lw.PPOCR.CTargets + RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") +install(TARGETS lw-recognize-ppm lw-rec-benchmark lw-ocr-benchmark lw-classify-ppm lw-detect-ppm + lw-ocr-ppm + RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") +install(FILES include/lw_infer.h + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +install(FILES + examples/CMakeLists.txt + examples/classify_ppm.c + examples/detect_ppm.c + examples/ocr_ppm.c + examples/abi_v1_client.c + examples/ppm_image.h + examples/recognize_ppm.c + DESTINATION examples) +install(DIRECTORY + examples/csharp-common + examples/csharp-winforms + examples/http-server + examples/java-jni + DESTINATION examples + PATTERN "bin" EXCLUDE + PATTERN "obj" EXCLUDE) +install(FILES + "${LW_PPOCR_C_REC_LWM}" + "${LW_PPOCR_C_CLS_LWM}" + "${LW_PPOCR_C_DET_LWM}" + "${LW_PPOCR_C_REC_ONNX}" "${LW_PPOCR_C_CLS_ONNX}" "${LW_PPOCR_C_DET_ONNX}" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/ppocr_keys.txt" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/model.json" + "${LW_PPOCR_C_REC_DEMO_PPM}" + "${LW_PPOCR_C_DET_DEMO_PPM}" + "${CMAKE_CURRENT_SOURCE_DIR}/models/ppocrv6-tiny/sample.jpg" + DESTINATION models) +endif() +if(LW_BUILD_CSHARP_DEMOS) + install(FILES + "${LW_MANAGED_WINFORMS_EXE}" + "${LW_MANAGED_WINFORMS_EXE}.config" + DESTINATION "${CMAKE_INSTALL_BINDIR}") +endif() +if(LW_BUILD_HTTP_DEMO) + install(TARGETS lw-http-ocr + RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") + install(DIRECTORY examples/http-server/www/ + DESTINATION www) +endif() +install(DIRECTORY docs/ + DESTINATION docs + FILES_MATCHING + PATTERN "*.md" + PATTERN "*.jpg") +install(FILES + abi/c-abi-v1-candidate.json + abi/runtime-contract-v1.json + abi/c-abi-v1-layout.json + abi/lwm-v0.1-layout.json + abi/web-abi-v1-candidate.json + abi/exports-v1-candidate.txt + DESTINATION docs/abi) +install(FILES + README.md + README.zh-CN.md + LICENSE + THIRD-PARTY-NOTICES.md + dependencies.lock.json + sbom.cdx.json + DESTINATION .) +install(FILES + ci/stable-release-scope.json + DESTINATION ci) +if(LW_BUILD_ENVIRONMENT_FILE) + if(NOT EXISTS "${LW_BUILD_ENVIRONMENT_FILE}") + message(FATAL_ERROR + "LW_BUILD_ENVIRONMENT_FILE does not exist: ${LW_BUILD_ENVIRONMENT_FILE}") + endif() + install(FILES "${LW_BUILD_ENVIRONMENT_FILE}" + DESTINATION . + RENAME BUILD-ENVIRONMENT.txt) +endif() +install(FILES + licenses/PaddleOCR-models-APACHE-2.0.txt + licenses/cpp-httplib-MIT.txt + DESTINATION licenses) + +if(NOT EMSCRIPTEN) +set(LW_CMAKE_PACKAGE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/lw.PPOCR.C") +configure_package_config_file( + cmake/lw.PPOCR.CConfig.cmake.in + "${CMAKE_CURRENT_BINARY_DIR}/lw.PPOCR.CConfig.cmake" + INSTALL_DESTINATION "${LW_CMAKE_PACKAGE_DIR}") +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/lw.PPOCR.CConfigVersion.cmake" + VERSION "${PROJECT_VERSION}" + COMPATIBILITY SameMajorVersion) +install(EXPORT lw.PPOCR.CTargets + FILE lw.PPOCR.CTargets.cmake + NAMESPACE lw.PPOCR.C:: + DESTINATION "${LW_CMAKE_PACKAGE_DIR}") +install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/lw.PPOCR.CConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/lw.PPOCR.CConfigVersion.cmake" + DESTINATION "${LW_CMAKE_PACKAGE_DIR}") +endif() + +if(BUILD_TESTING AND NOT EMSCRIPTEN) + add_test( + NAME staged_package + COMMAND "${CMAKE_COMMAND}" + -DLW_BUILD_DIR=${CMAKE_CURRENT_BINARY_DIR} + -DLW_STAGE_DIR=${CMAKE_CURRENT_BINARY_DIR}/stage-test + -DLW_CONFIG=$ + -DLW_GENERATOR=${CMAKE_GENERATOR} + -DLW_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} + -DLW_C_COMPILER=${CMAKE_C_COMPILER} + -DLW_RC_COMPILER=${CMAKE_RC_COMPILER} + -DLW_MT=${CMAKE_MT} + -DLW_PYTHON=${Python3_EXECUTABLE} + -DLW_TEST_SCRIPT=${CMAKE_CURRENT_SOURCE_DIR}/tests/test_staged_package.py + -DLW_HTTP_TEST_SCRIPT=${CMAKE_CURRENT_SOURCE_DIR}/tests/test_http_demo.py + -DLW_HTTP_TEST_REQUEST_TIMEOUT=${LW_HTTP_TEST_REQUEST_TIMEOUT} + -DLW_HTTP_TEST_PROCESS_TIMEOUT=${LW_HTTP_TEST_PROCESS_TIMEOUT} + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_staged_package_test.cmake") + set_tests_properties(staged_package PROPERTIES + TIMEOUT "${LW_STAGED_PACKAGE_TEST_TIMEOUT}") +endif() + +if(LW_PACKAGE_PLATFORM_OVERRIDE) + set(LW_PACKAGE_PLATFORM "${LW_PACKAGE_PLATFORM_OVERRIDE}") + if(WIN32) + set(CPACK_GENERATOR ZIP) + else() + set(CPACK_GENERATOR TGZ) + endif() +elseif(WIN32) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(LW_PACKAGE_PLATFORM windows-x64-msvc) + else() + set(LW_PACKAGE_PLATFORM windows-x86-msvc) + endif() + set(CPACK_GENERATOR ZIP) +else() + string(TOLOWER "${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}" + LW_PACKAGE_PLATFORM) + set(CPACK_GENERATOR TGZ) +endif() +set(CPACK_PACKAGE_NAME "lw.PPOCR.C") +set(CPACK_PACKAGE_VENDOR "天天代码码天天") +set(CPACK_PACKAGE_VERSION "${PROJECT_VERSION}") +set(CPACK_PACKAGE_FILE_NAME + "lw.PPOCR.C-${PROJECT_VERSION}-${LW_PACKAGE_PLATFORM}") +set(CPACK_PACKAGE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/packages") +set(CPACK_PACKAGE_CHECKSUM SHA256) +set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE") +include(CPack) diff --git a/deploy/ocr/vendor/lw.PPOCR.C/LICENSE b/deploy/ocr/vendor/lw.PPOCR.C/LICENSE new file mode 100644 index 00000000..64eecf39 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 天天代码码天天 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/deploy/ocr/vendor/lw.PPOCR.C/THIRD-PARTY-NOTICES.md b/deploy/ocr/vendor/lw.PPOCR.C/THIRD-PARTY-NOTICES.md new file mode 100644 index 00000000..091cfd73 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/THIRD-PARTY-NOTICES.md @@ -0,0 +1,74 @@ +# Third-party notices + +The pure-C OCR libraries have no large third-party runtime dependency. The +repository also contains an optional HTTP Demo, development tools, and model +inputs with their own licenses. + +## Optional HTTP Demo + +The cross-platform native HTTP OCR Demo embeds `cpp-httplib` 0.48.0 under the +MIT license. Its vendored header is the Windows 7-compatible copy from the +referenced `lw.OpenCVDNN.PPOCR` repository. See +`licenses/cpp-httplib-MIT.txt`. This dependency belongs to the optional C++ +HTTP executable; it is not linked into the pure-C OCR libraries. + +## PP-OCRv6 tiny model assets + +The ONNX model files, dictionary, and sample image under +`models/ppocrv6-tiny/` are derived from the PP-OCR/PaddleOCR ecosystem and are +included for direct runtime loading, converter development and reproducible analysis under the Apache +License 2.0. See `licenses/PaddleOCR-models-APACHE-2.0.txt`. + +## Native ONNX importer reference + +The bounded-reader approach was studied in `lxw112190/lw.PPOCR.Vulkan` +(`src/onnx_import.cpp`, Apache-2.0). The implementation in this repository is +written in C for our existing semantic IR; it does not include Vulkan's C++ +graph/shader backend or link a protobuf library. Model bytes and upstream +licenses are unchanged. Small/Medium assets use the same PaddleOCR model +license as Tiny. + +## Converter-only Python dependencies + +- ONNX — Apache-2.0 +- ONNX Runtime — MIT +- NumPy — BSD-3-Clause +- Pillow — HPND +- protobuf — BSD-3-Clause +- ml_dtypes — Apache-2.0 +- typing_extensions — PSF-2.0 + +These packages are development/converter/test dependencies. They are not linked +into the pure-C deployment runtime. + +## WASM build and browser-test dependencies + +- Emscripten 4.0.15 — MIT and University of Illinois/NCSA Open Source License; +- Playwright for Python 1.62.0 — Apache-2.0; +- Chromium downloaded by Playwright — CI smoke-test browser under its upstream + licenses and third-party notices. + +Emscripten is the build toolchain for the standalone HTML artifact. Playwright +and Chromium run only in CI to open that artifact, execute real OCR repeatedly, +and check that WASM memory remains stable. They are not embedded in the HTML or +linked into the native runtime. + +The default standalone HTML also embeds the legacy browser build of Mozilla +PDF.js 6.3.289 under the Apache License 2.0. PDF.js converts PDF pages to Canvas +pixels in the browser; it is not part of `lw-ppocr.js`, the native runtime, or +the public C ABI. The vendored distribution and license are under +`web/vendor/pdfjs/`. Configure with `LW_WEB_PDF=OFF` to omit it. + +The same HTML embeds PDF.js's optional `jbig2.wasm`, `openjpeg.wasm`, and +`qcms_bg.wasm` helpers. Their upstream notices are preserved as +`web/vendor/pdfjs/LICENSE_JBIG2`, `LICENSE_OPENJPEG`, `LICENSE_QCMS`, and the +corresponding `LICENSE_PDFJS_*` files. These resources enable offline +CCITT/JBIG2, JPX/JPEG2000, and ICC rendering paths. + +## Windows runtime files + +Windows binary archives include the Microsoft Visual C++ Runtime and Universal +C Runtime app-local files selected by CMake from the installed Visual Studio +toolchain. Their redistribution and use are governed by the applicable +Microsoft Visual Studio license terms. They are packaging dependencies, not +source dependencies of the pure-C OCR runtime. diff --git a/deploy/ocr/vendor/lw.PPOCR.C/VENDORED.md b/deploy/ocr/vendor/lw.PPOCR.C/VENDORED.md new file mode 100644 index 00000000..d2ea1d66 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/VENDORED.md @@ -0,0 +1,10 @@ +# Vendored source + +- Upstream: https://github.com/lxw112190/lw.PPOCR.C +- Commit: `db9383226a4a4c6a96ac594072c9abf0243329fb` +- Version: v1.2.1 preparation commit +- License: see `LICENSE` and `licenses/` + +Only the native runtime sources, HTTP demo, and PP-OCRv6 Tiny model assets are +vendored. Python converters, browser assets, tests, and Small/Medium models are +omitted. diff --git a/deploy/ocr/vendor/lw.PPOCR.C/examples/http-server/main.cpp b/deploy/ocr/vendor/lw.PPOCR.C/examples/http-server/main.cpp new file mode 100644 index 00000000..f15b59a8 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/examples/http-server/main.cpp @@ -0,0 +1,826 @@ +#ifndef LW_HTTP_THREAD_POOL_COUNT +# if (defined(_WIN32) && !defined(_WIN64)) || \ + (defined(__SIZEOF_POINTER__) && __SIZEOF_POINTER__ == 4) +# define LW_HTTP_THREAD_POOL_COUNT 2 +# else +# define LW_HTTP_THREAD_POOL_COUNT 8 +# endif +#endif +#define CPPHTTPLIB_THREAD_POOL_COUNT LW_HTTP_THREAD_POOL_COUNT + +#include "httplib.h" +#include "lw_infer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# include +#elif defined(__APPLE__) +# include +# include +#else +# include +#endif + +namespace { + +const uint64_t kMaxImagePixels = UINT64_C(40000000); +const size_t kMaxWebPageBytes = 4u * 1024u * 1024u; + +bool AllocationFits(uint64_t count, size_t element_size) { + return element_size != 0u && count <= static_cast(SIZE_MAX / element_size); +} + +uint32_t DefaultOcrWorkers() { + lw_ocr_options options; + lw_ocr_options_init(&options); + return options.worker_count; +} + +struct Config { + std::string host; + int port; + std::string models; + std::string web_root; + bool use_classifier; + uint32_t ocr_workers; + uint32_t rec_max_width; + + Config() : host("127.0.0.1"), port(8787), use_classifier(true), + ocr_workers(DefaultOcrWorkers()), rec_max_width(960u) {} +}; + +struct BgrImage { + uint32_t width; + uint32_t height; + std::vector pixels; + + BgrImage() : width(0u), height(0u) {} +}; + +struct OcrOutput { + uint32_t detected_count; + std::vector lines; + std::string text; + + OcrOutput() : detected_count(0u) {} +}; + +class ApiError : public std::runtime_error { +public: + ApiError(int status, const std::string& code, const std::string& message) + : std::runtime_error(message), status_(status), code_(code) {} + int status() const { return status_; } + const std::string& code() const { return code_; } + +private: + int status_; + std::string code_; +}; + +std::mutex g_log_mutex; +std::atomic g_request_sequence(0u); + +void Log(const std::string& message) { + std::lock_guard guard(g_log_mutex); + std::cout << message << std::endl; +} + +std::string JsonEscape(const std::string& value) { + std::ostringstream output; + for (size_t index = 0u; index < value.size(); ++index) { + const unsigned char character = + static_cast(value[index]); + switch (character) { + case '\"': output << "\\\""; break; + case '\\': output << "\\\\"; break; + case '\b': output << "\\b"; break; + case '\f': output << "\\f"; break; + case '\n': output << "\\n"; break; + case '\r': output << "\\r"; break; + case '\t': output << "\\t"; break; + default: + if (character < 0x20u) { + output << "\\u" << std::hex << std::setw(4) + << std::setfill('0') << static_cast(character) + << std::dec << std::setfill(' '); + } else { + output << static_cast(character); + } + break; + } + } + return output.str(); +} + +std::string MakeRequestId() { + const uint64_t ticks = static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + const uint64_t sequence = ++g_request_sequence; + std::ostringstream output; + output << std::hex << ticks << '-' << sequence; + return output.str(); +} + +std::string ParentPath(const std::string& path) { + const size_t slash = path.find_last_of("/\\"); + if (slash == std::string::npos) return "."; + if (slash == 0u) return path.substr(0u, 1u); + return path.substr(0u, slash); +} + +std::string JoinPath(const std::string& parent, const std::string& child) { + if (parent.empty()) return child; + if (parent == ".") return "./" + child; + const char last = parent[parent.size() - 1u]; + return parent + ((last == '/' || last == '\\') ? "" : "/") + child; +} + +// Choose by presence, not by a failed import: a damaged ONNX must report its +// error instead of silently falling back to a different model. +std::string ModelPath(const std::string& directory, const std::string& name) { + const std::string onnx = JoinPath(directory, name + ".onnx"); +#if defined(_WIN32) + const int count = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, + onnx.c_str(), -1, NULL, 0); + if (count > 0) { + std::vector wide(static_cast(count)); + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, onnx.c_str(), -1, + wide.data(), count) > 0) { + const DWORD attributes = GetFileAttributesW(wide.data()); + if (attributes != INVALID_FILE_ATTRIBUTES && + (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0u) return onnx; + } + } +#else + std::ifstream file(onnx.c_str(), std::ios::binary); + if (file.good()) return onnx; +#endif + return JoinPath(directory, name + ".lwm"); +} + +#if defined(_WIN32) +std::string WideToUtf8(const wchar_t* value) { + if (value == NULL || *value == L'\0') return std::string(); + const int count = WideCharToMultiByte( + CP_UTF8, WC_ERR_INVALID_CHARS, value, -1, NULL, 0, NULL, NULL); + if (count <= 0) throw std::runtime_error("UTF-16 to UTF-8 conversion failed"); + std::vector bytes(static_cast(count)); + if (WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, value, -1, + &bytes[0], count, NULL, NULL) <= 0) { + throw std::runtime_error("UTF-16 to UTF-8 conversion failed"); + } + return std::string(&bytes[0]); +} + +std::wstring Utf8ToWide(const std::string& value) { + if (value.empty()) return std::wstring(); + const int count = MultiByteToWideChar( + CP_UTF8, MB_ERR_INVALID_CHARS, value.c_str(), -1, NULL, 0); + if (count <= 0) throw std::runtime_error("UTF-8 path conversion failed"); + std::vector characters(static_cast(count)); + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, value.c_str(), -1, + &characters[0], count) <= 0) { + throw std::runtime_error("UTF-8 path conversion failed"); + } + return std::wstring(&characters[0]); +} +#endif + +std::string ExecutableDirectory(const char* argv0) { +#if defined(_WIN32) + std::vector path(32768u); + const DWORD count = GetModuleFileNameW( + NULL, &path[0], static_cast(path.size())); + if (count != 0u && count < path.size()) { + path[count] = L'\0'; + return ParentPath(WideToUtf8(&path[0])); + } +#elif defined(__APPLE__) + uint32_t size = 0u; + (void)_NSGetExecutablePath(NULL, &size); + if (size != 0u) { + std::vector path(size + 1u, '\0'); + if (_NSGetExecutablePath(&path[0], &size) == 0) { + return ParentPath(&path[0]); + } + } +#else + std::vector path(4096u, '\0'); + const ssize_t count = readlink("/proc/self/exe", &path[0], path.size() - 1u); + if (count > 0) { + path[static_cast(count)] = '\0'; + return ParentPath(&path[0]); + } +#endif + return ParentPath(argv0 == NULL ? "." : argv0); +} + +std::string ReadFileUtf8(const std::string& path, size_t maximum_size) { + FILE* file = NULL; +#if defined(_WIN32) + const std::wstring wide = Utf8ToWide(path); + if (_wfopen_s(&file, wide.c_str(), L"rb") != 0) file = NULL; +#else + file = std::fopen(path.c_str(), "rb"); +#endif + if (file == NULL) throw std::runtime_error("unable to open file: " + path); + std::vector data; + char buffer[8192]; + for (;;) { + const size_t count = std::fread(buffer, 1u, sizeof(buffer), file); + if (count != 0u) { + if (data.size() > maximum_size - count) { + std::fclose(file); + throw std::runtime_error("file is too large: " + path); + } + data.insert(data.end(), buffer, buffer + count); + } + if (count < sizeof(buffer)) { + if (std::ferror(file) != 0) { + std::fclose(file); + throw std::runtime_error("unable to read file: " + path); + } + break; + } + } + std::fclose(file); + return std::string(data.begin(), data.end()); +} + +bool IsSpace(unsigned char value) { + return value == ' ' || value == '\t' || value == '\r' || value == '\n' || + value == '\f' || value == '\v'; +} + +std::string ReadPpmToken(const std::string& bytes, size_t* offset) { + if (offset == NULL) throw ApiError(400, "bad_request", "invalid PPM parser state"); + for (;;) { + while (*offset < bytes.size() && + IsSpace(static_cast(bytes[*offset]))) ++(*offset); + if (*offset >= bytes.size() || bytes[*offset] != '#') break; + while (*offset < bytes.size() && bytes[*offset] != '\n') ++(*offset); + } + const size_t begin = *offset; + while (*offset < bytes.size() && + !IsSpace(static_cast(bytes[*offset]))) ++(*offset); + if (*offset == begin || *offset >= bytes.size()) { + throw ApiError(400, "bad_request", "invalid P6 PPM header"); + } + const std::string token = bytes.substr(begin, *offset - begin); + const unsigned char delimiter = static_cast(bytes[*offset]); + ++(*offset); + if (delimiter == '\r' && *offset < bytes.size() && bytes[*offset] == '\n') { + ++(*offset); + } + return token; +} + +uint32_t ParsePositiveU32(const std::string& value, const char* field) { + if (value.empty()) throw ApiError(400, "bad_request", std::string("invalid ") + field); + uint64_t parsed = 0u; + for (size_t index = 0u; index < value.size(); ++index) { + const unsigned char character = static_cast(value[index]); + if (character < '0' || character > '9') { + throw ApiError(400, "bad_request", std::string("invalid ") + field); + } + const uint64_t digit = static_cast(character - '0'); + if (parsed > (UINT32_MAX - digit) / 10u) { + throw ApiError(400, "bad_request", std::string("invalid ") + field); + } + parsed = parsed * 10u + digit; + } + if (parsed == 0u) throw ApiError(400, "bad_request", std::string("invalid ") + field); + return static_cast(parsed); +} + +BgrImage DecodeP6Ppm(const std::string& bytes) { + size_t offset = 0u; + if (ReadPpmToken(bytes, &offset) != "P6") { + throw ApiError(415, "unsupported_media_type", + "server accepts binary P6 PPM; the browser Demo converts JPEG/PNG before upload"); + } + BgrImage image; + image.width = ParsePositiveU32(ReadPpmToken(bytes, &offset), "PPM width"); + image.height = ParsePositiveU32(ReadPpmToken(bytes, &offset), "PPM height"); + if (ParsePositiveU32(ReadPpmToken(bytes, &offset), "PPM max value") != 255u) { + throw ApiError(400, "bad_request", "P6 PPM max value must be 255"); + } + const uint64_t pixels = static_cast(image.width) * image.height; + if (pixels > kMaxImagePixels || pixels > SIZE_MAX / 3u) { + throw ApiError(413, "image_too_large", "decoded image exceeds 40,000,000 pixels"); + } + const size_t byte_count = static_cast(pixels * 3u); + if (offset > bytes.size() || bytes.size() - offset != byte_count) { + throw ApiError(400, "bad_request", "P6 PPM pixel payload length is invalid"); + } + image.pixels.resize(byte_count); + for (size_t index = 0u; index < byte_count; index += 3u) { + image.pixels[index] = static_cast(bytes[offset + index + 2u]); + image.pixels[index + 1u] = static_cast(bytes[offset + index + 1u]); + image.pixels[index + 2u] = static_cast(bytes[offset + index]); + } + return image; +} + +uint16_t ReadLe16(const std::string& bytes, size_t offset, const char* field) { + if (offset > bytes.size() || bytes.size() - offset < 2u) { + throw ApiError(400, "bad_request", std::string("invalid BMP ") + field); + } + return static_cast( + static_cast(bytes[offset]) | + (static_cast(static_cast(bytes[offset + 1u])) << 8u)); +} + +uint32_t ReadLe32(const std::string& bytes, size_t offset, const char* field) { + if (offset > bytes.size() || bytes.size() - offset < 4u) { + throw ApiError(400, "bad_request", std::string("invalid BMP ") + field); + } + return static_cast(static_cast(bytes[offset])) | + (static_cast(static_cast(bytes[offset + 1u])) << 8u) | + (static_cast(static_cast(bytes[offset + 2u])) << 16u) | + (static_cast(static_cast(bytes[offset + 3u])) << 24u); +} + +BgrImage DecodeBmp24(const std::string& bytes) { + if (bytes.size() < 54u || bytes[0] != 'B' || bytes[1] != 'M') { + throw ApiError(400, "bad_request", "invalid BMP header"); + } + const uint32_t pixel_offset = ReadLe32(bytes, 10u, "pixel offset"); + const uint32_t dib_size = ReadLe32(bytes, 14u, "DIB header"); + const uint32_t raw_width = ReadLe32(bytes, 18u, "width"); + const uint32_t raw_height = ReadLe32(bytes, 22u, "height"); + const int64_t signed_width = raw_width <= INT32_MAX + ? static_cast(raw_width) + : static_cast(raw_width) - INT64_C(4294967296); + const int64_t signed_height = raw_height <= INT32_MAX + ? static_cast(raw_height) + : static_cast(raw_height) - INT64_C(4294967296); + const uint16_t planes = ReadLe16(bytes, 26u, "planes"); + const uint16_t bits_per_pixel = ReadLe16(bytes, 28u, "bit depth"); + const uint32_t compression = ReadLe32(bytes, 30u, "compression"); + if (dib_size < 40u || dib_size > bytes.size() - 14u || + pixel_offset < 14u + static_cast(dib_size) || + planes != 1u || bits_per_pixel != 24u || + compression != 0u || signed_width <= 0 || signed_height == 0) { + throw ApiError(415, "unsupported_media_type", + "server accepts uncompressed 24-bit BMP images"); + } + BgrImage image; + image.width = static_cast(signed_width); + image.height = static_cast( + signed_height < 0 ? -signed_height : signed_height); + const uint64_t pixels = static_cast(image.width) * image.height; + if (pixels > kMaxImagePixels || !AllocationFits(pixels, 3u)) { + throw ApiError(413, "image_too_large", "decoded image exceeds 40,000,000 pixels"); + } + const uint64_t row_bytes = + (static_cast(image.width) * 3u + 3u) & ~UINT64_C(3); + const uint64_t payload_bytes = row_bytes * image.height; + if (pixel_offset > bytes.size() || payload_bytes > bytes.size() - pixel_offset) { + throw ApiError(400, "bad_request", "BMP pixel payload length is invalid"); + } + image.pixels.resize(static_cast(pixels * 3u)); + for (uint32_t y = 0u; y < image.height; ++y) { + const uint32_t source_y = + signed_height < 0 ? y : image.height - 1u - y; + const size_t source_offset = + static_cast(pixel_offset + row_bytes * source_y); + const size_t destination_offset = + static_cast(image.width) * y * 3u; + std::memcpy(&image.pixels[destination_offset], + bytes.data() + source_offset, static_cast(image.width) * 3u); + } + return image; +} + +int Base64Value(unsigned char character) { + if (character >= 'A' && character <= 'Z') return character - 'A'; + if (character >= 'a' && character <= 'z') return character - 'a' + 26; + if (character >= '0' && character <= '9') return character - '0' + 52; + if (character == '+') return 62; + if (character == '/') return 63; + return -1; +} + +std::string DecodeBase64(std::string value) { + const size_t comma = value.find(','); + if (value.compare(0u, 5u, "data:") == 0 && comma != std::string::npos) { + value.erase(0u, comma + 1u); + } + value.erase(std::remove_if(value.begin(), value.end(), + [](char character) { + return IsSpace(static_cast(character)); + }), value.end()); + if (value.empty() || value.size() % 4u != 0u) { + throw ApiError(400, "bad_request", "imageBase64 is invalid"); + } + std::string decoded; + decoded.reserve(value.size() / 4u * 3u); + for (size_t index = 0u; index < value.size(); index += 4u) { + const bool final_group = index + 4u == value.size(); + const int a = Base64Value(static_cast(value[index])); + const int b = Base64Value(static_cast(value[index + 1u])); + const int c = value[index + 2u] == '=' ? -2 : + Base64Value(static_cast(value[index + 2u])); + const int d = value[index + 3u] == '=' ? -2 : + Base64Value(static_cast(value[index + 3u])); + if (a < 0 || b < 0 || c == -1 || d == -1 || + (!final_group && (c == -2 || d == -2)) || + (c == -2 && d != -2)) { + throw ApiError(400, "bad_request", "imageBase64 is invalid"); + } + const uint32_t packed = (static_cast(a) << 18u) | + (static_cast(b) << 12u) | + (static_cast(c < 0 ? 0 : c) << 6u) | + static_cast(d < 0 ? 0 : d); + decoded.push_back(static_cast((packed >> 16u) & 0xffu)); + if (c >= 0) decoded.push_back(static_cast((packed >> 8u) & 0xffu)); + if (d >= 0) decoded.push_back(static_cast(packed & 0xffu)); + } + return decoded; +} + +std::string ExtractJsonString(const std::string& json, const std::string& key) { + const std::string marker = "\"" + key + "\""; + size_t offset = json.find(marker); + if (offset == std::string::npos) return std::string(); + offset += marker.size(); + while (offset < json.size() && IsSpace(static_cast(json[offset]))) ++offset; + if (offset >= json.size() || json[offset++] != ':') return std::string(); + while (offset < json.size() && IsSpace(static_cast(json[offset]))) ++offset; + if (offset >= json.size() || json[offset++] != '\"') return std::string(); + std::string value; + while (offset < json.size()) { + const char character = json[offset++]; + if (character == '\"') return value; + if (character == '\\') { + if (offset >= json.size()) break; + const char escaped = json[offset++]; + if (escaped == '/' || escaped == '\\' || escaped == '\"') { + value.push_back(escaped); + } else { + throw ApiError(400, "bad_request", "imageBase64 contains an invalid JSON escape"); + } + } else { + value.push_back(character); + } + } + throw ApiError(400, "bad_request", "JSON string is incomplete"); +} + +std::string LowerMediaType(const std::string& value) { + const size_t semicolon = value.find(';'); + std::string type = value.substr(0u, semicolon); + std::transform(type.begin(), type.end(), type.begin(), + [](char character) { + const unsigned char byte = static_cast(character); + return static_cast(byte >= 'A' && byte <= 'Z' ? byte + 32u : byte); + }); + while (!type.empty() && IsSpace(static_cast(type.front()))) type.erase(type.begin()); + while (!type.empty() && IsSpace(static_cast(type.back()))) type.pop_back(); + return type; +} + +BgrImage ReadRequestImage(const httplib::Request& request) { + const std::string content_type = LowerMediaType( + request.get_header_value("Content-Type")); + if (content_type == "application/json") { + std::string encoded = ExtractJsonString(request.body, "imageBase64"); + if (encoded.empty()) encoded = ExtractJsonString(request.body, "image_base64"); + if (encoded.empty()) { + throw ApiError(400, "bad_request", "JSON must contain imageBase64"); + } + return DecodeP6Ppm(DecodeBase64(encoded)); + } + if (content_type == "image/x-portable-pixmap" || + content_type == "image/x-portable-anymap" || + content_type == "application/octet-stream") { + return DecodeP6Ppm(request.body); + } + if (content_type == "image/bmp" || content_type == "image/x-ms-bmp") { + return DecodeBmp24(request.body); + } + throw ApiError(415, "unsupported_media_type", + "use image/x-portable-pixmap, image/bmp, application/octet-stream, " + "or JSON/Base64 P6 PPM"); +} + +class OcrEngine { +public: + OcrEngine(const Config& config) : handle_(NULL) { + lw_ocr_options options; + lw_error error; + lw_ocr_options_init(&options); + options.use_direction_classification = config.use_classifier ? 1u : 0u; + options.worker_count = config.ocr_workers; + options.recognizer.target_width = config.rec_max_width; + lw_error_init(&error); + const std::string detector = ModelPath(config.models, "det"); + const std::string classifier = ModelPath(config.models, "cls"); + const std::string recognizer = ModelPath(config.models, "rec"); + const std::string dictionary = JoinPath(config.models, "ppocr_keys.txt"); + const lw_status status = lw_ocr_create( + detector.c_str(), config.use_classifier ? classifier.c_str() : NULL, + recognizer.c_str(), dictionary.c_str(), &options, &handle_, &error); + if (status != LW_STATUS_OK || handle_ == NULL) { + throw std::runtime_error(std::string("OCR initialization failed: ") + + lw_status_string(status) + ": " + error.message); + } + lw_ocr_info_init(&info_); + if (lw_ocr_get_info(handle_, &info_) != LW_STATUS_OK || + info_.max_line_capacity == 0u || info_.max_text_capacity == 0u || + !AllocationFits(info_.max_line_capacity, sizeof(lw_ocr_line)) || + !AllocationFits(info_.max_text_capacity, sizeof(char))) { + lw_ocr_free(handle_); + handle_ = NULL; + throw std::runtime_error("unable to query OCR output capacities"); + } + lines_.resize(info_.max_line_capacity); + text_.resize(static_cast(info_.max_text_capacity)); + } + + ~OcrEngine() { lw_ocr_free(handle_); } + + OcrOutput Run(const BgrImage& image) { + std::lock_guard guard(mutex_); + lw_ocr_result result; + lw_error error; + lw_ocr_result_init(&result); + lw_error_init(&error); + const lw_status status = lw_ocr_run_bgr_u8( + handle_, image.pixels.empty() ? NULL : &image.pixels[0], + static_cast(image.pixels.size()), image.width, image.height, + image.width * 3u, &lines_[0], static_cast(lines_.size()), + &text_[0], static_cast(text_.size()), &result, &error); + if (status != LW_STATUS_OK) { + throw ApiError(422, "ocr_failed", std::string(lw_status_string(status)) + + ": " + error.message); + } + if (result.line_count > lines_.size() || + result.required_text_capacity > text_.size()) { + throw std::runtime_error("native OCR returned invalid output capacities"); + } + OcrOutput output; + output.detected_count = result.detected_count; + output.lines.assign(lines_.begin(), lines_.begin() + result.line_count); + output.text.assign(text_.begin(), + text_.begin() + static_cast(result.required_text_capacity)); + return output; + } + +private: + OcrEngine(const OcrEngine&); + OcrEngine& operator=(const OcrEngine&); + lw_ocr* handle_; + lw_ocr_info info_; + std::vector lines_; + std::vector text_; + std::mutex mutex_; +}; + +std::string BuildOcrJson(const std::string& request_id, const BgrImage& image, + const OcrOutput& output, double elapsed_ms) { + std::ostringstream json; + json.imbue(std::locale::classic()); + json << std::setprecision(9) << "{\"ok\":true,\"api_version\":1," + << "\"request_id\":\"" << JsonEscape(request_id) << "\"," + << "\"image_width\":" << image.width << ",\"image_height\":" + << image.height << ",\"detected_count\":" << output.detected_count + << ",\"result\":["; + for (size_t index = 0u; index < output.lines.size(); ++index) { + const lw_ocr_line& line = output.lines[index]; + if (line.text_offset > output.text.size() || + line.text_length > output.text.size() - static_cast(line.text_offset)) { + throw std::runtime_error("native OCR returned an invalid text range"); + } + const std::string text = output.text.substr( + static_cast(line.text_offset), static_cast(line.text_length)); + if (index != 0u) json << ','; + json << "{\"text\":\"" << JsonEscape(text) << "\"," + << "\"score\":" << line.recognition_score << ',' + << "\"det_score\":" << line.box.score << ',' + << "\"cls_label\":" << line.classification_label << ',' + << "\"cls_score\":" << line.classification_score << ',' + << "\"rotation\":" << line.applied_rotation_degrees << ',' + << "\"x1\":" << line.box.x1 << ",\"y1\":" << line.box.y1 << ',' + << "\"x2\":" << line.box.x2 << ",\"y2\":" << line.box.y2 << ',' + << "\"x3\":" << line.box.x3 << ",\"y3\":" << line.box.y3 << ',' + << "\"x4\":" << line.box.x4 << ",\"y4\":" << line.box.y4 << '}'; + } + json << "],\"timing_ms\":{\"server_total\":" << elapsed_ms << "}}"; + return json.str(); +} + +void SetCommonHeaders(httplib::Response& response, const std::string& request_id) { + response.set_header("Cache-Control", "no-store"); + response.set_header("X-Content-Type-Options", "nosniff"); + response.set_header("X-API-Version", "1"); + response.set_header("X-Request-ID", request_id); +} + +void SetJson(httplib::Response& response, int status, const std::string& body, + const std::string& request_id) { + response.status = status; + SetCommonHeaders(response, request_id); + response.set_content(body, "application/json; charset=utf-8"); +} + +void SetError(httplib::Response& response, int status, const std::string& code, + const std::string& message, const std::string& request_id) { + std::ostringstream json; + json << "{\"ok\":false,\"api_version\":1,\"request_id\":\"" + << JsonEscape(request_id) << "\",\"error_code\":\"" + << JsonEscape(code) << "\",\"error\":\"" + << JsonEscape(message) << "\"}"; + SetJson(response, status, json.str(), request_id); +} + +int ParsePort(const std::string& value) { + const uint32_t parsed = ParsePositiveU32(value, "port"); + if (parsed > 65535u) throw std::runtime_error("port must be between 1 and 65535"); + return static_cast(parsed); +} + +Config ParseArguments(const std::vector& arguments) { + Config config; + const std::string executable_directory = ExecutableDirectory( + arguments.empty() ? NULL : arguments[0].c_str()); + const std::string package_root = ParentPath(executable_directory); + config.models = JoinPath(package_root, "models"); + config.web_root = JoinPath(package_root, "www"); + for (size_t index = 1u; index < arguments.size(); ++index) { + const std::string& argument = arguments[index]; + if (argument == "--no-cls") { + config.use_classifier = false; + } else if (argument == "--host" || argument == "--port" || + argument == "--models" || argument == "--www" || + argument == "--ocr-workers" || argument == "--rec-max-width") { + if (index + 1u >= arguments.size()) { + throw std::runtime_error("missing value after " + argument); + } + const std::string value = arguments[++index]; + if (argument == "--host") config.host = value; + else if (argument == "--port") config.port = ParsePort(value); + else if (argument == "--models") config.models = value; + else if (argument == "--www") config.web_root = value; + else if (argument == "--ocr-workers") { + config.ocr_workers = ParsePositiveU32(value, "ocr-workers"); + if (config.ocr_workers > 16u) + throw std::runtime_error("ocr-workers must be between 1 and 16"); + } else { + config.rec_max_width = ParsePositiveU32(value, "rec-max-width"); + if (config.rec_max_width != 192u && config.rec_max_width != 320u && + config.rec_max_width != 480u && config.rec_max_width != 640u && + config.rec_max_width != 960u) { + throw std::runtime_error( + "rec-max-width must be one of 192, 320, 480, 640, or 960"); + } + } + } else if (argument == "--help" || argument == "-h") { + std::cout << "Usage: lw.PPOCR.C.HttpServer [--host ADDRESS] [--port PORT] " + "[--models DIRECTORY] [--www DIRECTORY] [--ocr-workers COUNT] " + "[--rec-max-width WIDTH] [--no-cls]\n"; + std::exit(0); + } else { + throw std::runtime_error("unknown argument: " + argument); + } + } + if (config.host.empty()) throw std::runtime_error("host must not be empty"); + return config; +} + +int Run(const std::vector& arguments) { + const Config config = ParseArguments(arguments); + OcrEngine engine(config); + const std::string index_html = ReadFileUtf8( + JoinPath(config.web_root, "index.html"), kMaxWebPageBytes); + httplib::Server server; + server.set_read_timeout(30, 0); + server.set_write_timeout(30, 0); + server.set_payload_max_length(sizeof(void*) == 4u ? + 10u * 1024u * 1024u : 50u * 1024u * 1024u); + + server.Get("/", [&index_html](const httplib::Request&, httplib::Response& response) { + const std::string request_id = MakeRequestId(); + SetCommonHeaders(response, request_id); + response.set_content(index_html, "text/html; charset=utf-8"); + }); + server.Get("/index.html", [&index_html](const httplib::Request&, httplib::Response& response) { + const std::string request_id = MakeRequestId(); + SetCommonHeaders(response, request_id); + response.set_content(index_html, "text/html; charset=utf-8"); + }); + server.Get("/health", [](const httplib::Request&, httplib::Response& response) { + const std::string request_id = MakeRequestId(); + SetJson(response, 200, + "{\"ok\":true,\"api_version\":1,\"request_id\":\"" + + JsonEscape(request_id) + "\",\"status\":\"ready\"}", request_id); + }); + server.Post("/api/ocr", [&engine](const httplib::Request& request, + httplib::Response& response) { + const std::string request_id = MakeRequestId(); + const std::chrono::steady_clock::time_point started = + std::chrono::steady_clock::now(); + try { + const BgrImage image = ReadRequestImage(request); + const OcrOutput output = engine.Run(image); + const double elapsed_ms = + std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + SetJson(response, 200, + BuildOcrJson(request_id, image, output, elapsed_ms), request_id); + std::ostringstream log; + log << "request_id=" << request_id << " status=200 duration_ms=" + << std::fixed << std::setprecision(1) << elapsed_ms + << " lines=" << output.lines.size(); + Log(log.str()); + } catch (const ApiError& error) { + SetError(response, error.status(), error.code(), error.what(), request_id); + Log("request_id=" + request_id + " status=" + + std::to_string(error.status()) + " error_code=" + error.code()); + } catch (const std::exception& error) { + SetError(response, 500, "internal_error", "server internal error", request_id); + Log("request_id=" + request_id + " status=500 detail=" + error.what()); + } + }); + server.set_error_handler([](const httplib::Request&, httplib::Response& response) { + if (response.has_header("X-API-Version")) return; + const std::string request_id = MakeRequestId(); + const int status = response.status == 413 ? 413 : + (response.status == 405 ? 405 : 404); + const std::string code = status == 413 ? "payload_too_large" : + (status == 405 ? "method_not_allowed" : "not_found"); + const std::string message = status == 413 ? "request body is too large" : + (status == 405 ? "method is not allowed" : "path does not exist"); + SetError(response, status, code, message, request_id); + }); + server.set_exception_handler([](const httplib::Request&, httplib::Response& response, + std::exception_ptr) { + const std::string request_id = MakeRequestId(); + SetError(response, 500, "internal_error", "server internal error", request_id); + }); + + std::ostringstream startup; + startup << "lw.PPOCR.C HTTP OCR Demo\n" + << "HTTP: http://" << config.host << ':' << config.port << "/\n" + << "models: " << config.models << "\n" + << "www: " << config.web_root << "\n" + << "OCR line workers: " << config.ocr_workers << "\n" + << "REC maximum width: " << config.rec_max_width << "\n" + << "HTTP workers: " << LW_HTTP_THREAD_POOL_COUNT; + Log(startup.str()); + const bool listening = server.listen(config.host.c_str(), config.port); + if (!listening) { + std::cerr << "unable to listen on " << config.host << ':' << config.port << std::endl; + return 2; + } + return 0; +} + +} // namespace + +#if defined(_WIN32) +int wmain(int argc, wchar_t** argv) { + SetConsoleOutputCP(CP_UTF8); + try { + std::vector arguments; + for (int index = 0; index < argc; ++index) { + arguments.push_back(WideToUtf8(argv[index])); + } + return Run(arguments); + } catch (const std::exception& error) { + std::cerr << "fatal: " << error.what() << std::endl; + return 1; + } +} +#else +int main(int argc, char** argv) { + try { + std::vector arguments; + for (int index = 0; index < argc; ++index) arguments.push_back(argv[index]); + return Run(arguments); + } catch (const std::exception& error) { + std::cerr << "fatal: " << error.what() << std::endl; + return 1; + } +} +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/examples/http-server/third_party/httplib.h b/deploy/ocr/vendor/lw.PPOCR.C/examples/http-server/third_party/httplib.h new file mode 100644 index 00000000..f3d29952 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/examples/http-server/third_party/httplib.h @@ -0,0 +1,20584 @@ +// +// httplib.h +// +// Copyright (c) 2026 Yuji Hirose. All rights reserved. +// MIT License +// + +#ifndef CPPHTTPLIB_HTTPLIB_H +#define CPPHTTPLIB_HTTPLIB_H + +#define CPPHTTPLIB_VERSION "0.48.0" +#define CPPHTTPLIB_VERSION_NUM "0x003000" + +// Win7 compatibility build: the Windows 10-only asynchronous DNS branch is +// disabled because CPPHTTPLIB_USE_NON_BLOCKING_GETADDRINFO is not defined. + +/* + * Configuration + */ + +#ifndef CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND +#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_KEEPALIVE_TIMEOUT_CHECK_INTERVAL_USECOND +#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_CHECK_INTERVAL_USECOND 10000 +#endif + +#ifndef CPPHTTPLIB_KEEPALIVE_MAX_COUNT +#define CPPHTTPLIB_KEEPALIVE_MAX_COUNT 100 +#endif + +#ifndef CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND +#define CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND 300 +#endif + +#ifndef CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND +#define CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_SERVER_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_SERVER_READ_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_SERVER_READ_TIMEOUT_USECOND +#define CPPHTTPLIB_SERVER_READ_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_SERVER_WRITE_TIMEOUT_SECOND +#define CPPHTTPLIB_SERVER_WRITE_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_SERVER_WRITE_TIMEOUT_USECOND +#define CPPHTTPLIB_SERVER_WRITE_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_CLIENT_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_CLIENT_READ_TIMEOUT_SECOND 300 +#endif + +#ifndef CPPHTTPLIB_CLIENT_READ_TIMEOUT_USECOND +#define CPPHTTPLIB_CLIENT_READ_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND +#define CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND +#define CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_CLIENT_MAX_TIMEOUT_MSECOND +#define CPPHTTPLIB_CLIENT_MAX_TIMEOUT_MSECOND 0 +#endif + +#ifndef CPPHTTPLIB_EXPECT_100_THRESHOLD +#define CPPHTTPLIB_EXPECT_100_THRESHOLD 1024 +#endif + +#ifndef CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND +#define CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND 1000 +#endif + +#ifndef CPPHTTPLIB_WAIT_EARLY_SERVER_RESPONSE_THRESHOLD +#define CPPHTTPLIB_WAIT_EARLY_SERVER_RESPONSE_THRESHOLD (1024 * 1024) +#endif + +#ifndef CPPHTTPLIB_WAIT_EARLY_SERVER_RESPONSE_TIMEOUT_MSECOND +#define CPPHTTPLIB_WAIT_EARLY_SERVER_RESPONSE_TIMEOUT_MSECOND 50 +#endif + +#ifndef CPPHTTPLIB_IDLE_INTERVAL_SECOND +#define CPPHTTPLIB_IDLE_INTERVAL_SECOND 0 +#endif + +#ifndef CPPHTTPLIB_IDLE_INTERVAL_USECOND +#ifdef _WIN32 +#define CPPHTTPLIB_IDLE_INTERVAL_USECOND 1000 +#else +#define CPPHTTPLIB_IDLE_INTERVAL_USECOND 0 +#endif +#endif + +#ifndef CPPHTTPLIB_REQUEST_URI_MAX_LENGTH +#define CPPHTTPLIB_REQUEST_URI_MAX_LENGTH 8192 +#endif + +#ifndef CPPHTTPLIB_HEADER_MAX_LENGTH +#define CPPHTTPLIB_HEADER_MAX_LENGTH 8192 +#endif + +#ifndef CPPHTTPLIB_HEADER_MAX_COUNT +#define CPPHTTPLIB_HEADER_MAX_COUNT 100 +#endif + +#ifndef CPPHTTPLIB_REDIRECT_MAX_COUNT +#define CPPHTTPLIB_REDIRECT_MAX_COUNT 20 +#endif + +#ifndef CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT +#define CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT 1024 +#endif + +#ifndef CPPHTTPLIB_PAYLOAD_MAX_LENGTH +#define CPPHTTPLIB_PAYLOAD_MAX_LENGTH (100 * 1024 * 1024) // 100MB +#endif + +#ifndef CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH +#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 8192 +#endif + +#ifndef CPPHTTPLIB_RANGE_MAX_COUNT +#define CPPHTTPLIB_RANGE_MAX_COUNT 1024 +#endif + +#ifndef CPPHTTPLIB_TCP_NODELAY +#define CPPHTTPLIB_TCP_NODELAY false +#endif + +#ifndef CPPHTTPLIB_IPV6_V6ONLY +#define CPPHTTPLIB_IPV6_V6ONLY false +#endif + +#ifndef CPPHTTPLIB_RECV_BUFSIZ +#define CPPHTTPLIB_RECV_BUFSIZ size_t(16384u) +#endif + +#ifndef CPPHTTPLIB_SEND_BUFSIZ +#define CPPHTTPLIB_SEND_BUFSIZ size_t(16384u) +#endif + +#ifndef CPPHTTPLIB_COMPRESSION_BUFSIZ +#define CPPHTTPLIB_COMPRESSION_BUFSIZ size_t(16384u) +#endif + +#ifndef CPPHTTPLIB_THREAD_POOL_COUNT +#define CPPHTTPLIB_THREAD_POOL_COUNT \ + ((std::max)(8u, std::thread::hardware_concurrency() > 0 \ + ? std::thread::hardware_concurrency() - 1 \ + : 0)) +#endif + +#ifndef CPPHTTPLIB_THREAD_POOL_MAX_COUNT +#define CPPHTTPLIB_THREAD_POOL_MAX_COUNT (CPPHTTPLIB_THREAD_POOL_COUNT * 4) +#endif + +#ifndef CPPHTTPLIB_THREAD_POOL_IDLE_TIMEOUT +#define CPPHTTPLIB_THREAD_POOL_IDLE_TIMEOUT 3 // seconds +#endif + +#ifndef CPPHTTPLIB_RECV_FLAGS +#define CPPHTTPLIB_RECV_FLAGS 0 +#endif + +#ifndef CPPHTTPLIB_SEND_FLAGS +#define CPPHTTPLIB_SEND_FLAGS 0 +#endif + +#ifndef CPPHTTPLIB_LISTEN_BACKLOG +#define CPPHTTPLIB_LISTEN_BACKLOG 5 +#endif + +#ifndef CPPHTTPLIB_MAX_LINE_LENGTH +#define CPPHTTPLIB_MAX_LINE_LENGTH 32768 +#endif + +#ifndef CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH +#define CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH 16777216 +#endif + +#ifndef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND 300 +#endif + +#ifndef CPPHTTPLIB_WEBSOCKET_CLOSE_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_CLOSE_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_WEBSOCKET_PING_INTERVAL_SECOND +#define CPPHTTPLIB_WEBSOCKET_PING_INTERVAL_SECOND 30 +#endif + +#ifndef CPPHTTPLIB_WEBSOCKET_MAX_MISSED_PONGS +#define CPPHTTPLIB_WEBSOCKET_MAX_MISSED_PONGS 0 +#endif + +/* + * Headers + */ + +#ifdef _WIN32 +#ifndef _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS +#endif //_CRT_SECURE_NO_WARNINGS + +#ifndef _CRT_NONSTDC_NO_DEPRECATE +#define _CRT_NONSTDC_NO_DEPRECATE +#endif //_CRT_NONSTDC_NO_DEPRECATE + +#if defined(_MSC_VER) +#if _MSC_VER < 1900 +#error Sorry, Visual Studio versions prior to 2015 are not supported +#endif + +#pragma comment(lib, "ws2_32.lib") + +#ifndef _SSIZE_T_DEFINED +using ssize_t = __int64; +#define _SSIZE_T_DEFINED +#endif +#endif // _MSC_VER + +#ifndef S_ISREG +#define S_ISREG(m) (((m) & S_IFREG) == S_IFREG) +#endif // S_ISREG + +#ifndef S_ISDIR +#define S_ISDIR(m) (((m) & S_IFDIR) == S_IFDIR) +#endif // S_ISDIR + +#ifndef NOMINMAX +#define NOMINMAX +#endif // NOMINMAX + +#include +#include +#include + +#if defined(__has_include) +#if __has_include() +// afunix.h uses types declared in winsock2.h, so has to be included after it. +#include +#define CPPHTTPLIB_HAVE_AFUNIX_H 1 +#endif +#endif + +#ifndef WSA_FLAG_NO_HANDLE_INHERIT +#define WSA_FLAG_NO_HANDLE_INHERIT 0x80 +#endif + +using nfds_t = unsigned long; +using socket_t = SOCKET; +using socklen_t = int; + +#else // not _WIN32 + +#include +#if !defined(_AIX) && !defined(__MVS__) +#include +#endif +#ifdef __MVS__ +#include +#ifndef NI_MAXHOST +#define NI_MAXHOST 1025 +#endif +#endif +#include +#include +#include +#ifdef __linux__ +#include +#undef _res // Undefine _res macro to avoid conflicts with user code (#2278) +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +using socket_t = int; +#ifndef INVALID_SOCKET +#define INVALID_SOCKET (-1) +#endif +#endif //_WIN32 + +#if defined(__APPLE__) +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// On macOS with a TLS backend, enable Keychain root certificates by default +// unless the user explicitly opts out. Not enabled on iOS/tvOS/watchOS since +// the SecTrustSettings APIs used to enumerate anchor certificates are macOS +// only; on those platforms the user must provide a CA bundle explicitly. +#if defined(__APPLE__) && defined(__clang__) && \ + !defined(CPPHTTPLIB_DISABLE_MACOSX_AUTOMATIC_ROOT_CERTIFICATES) && \ + (defined(CPPHTTPLIB_OPENSSL_SUPPORT) || \ + defined(CPPHTTPLIB_MBEDTLS_SUPPORT) || \ + defined(CPPHTTPLIB_WOLFSSL_SUPPORT)) +#if TARGET_OS_OSX +#ifndef CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN +#define CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN +#endif +#endif +#endif + +#if defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && \ + defined(__APPLE__) && !TARGET_OS_OSX +#error \ + "CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN is only supported on macOS. On iOS/tvOS/watchOS, supply a CA bundle via set_ca_cert_path()." +#endif + +// On Windows, enable Schannel certificate verification by default +// unless the user explicitly opts out. +#if defined(_WIN32) && \ + !defined(CPPHTTPLIB_DISABLE_WINDOWS_AUTOMATIC_ROOT_CERTIFICATES_UPDATE) +#define CPPHTTPLIB_WINDOWS_AUTOMATIC_ROOT_CERTIFICATES_UPDATE +#endif + +#if defined(CPPHTTPLIB_USE_NON_BLOCKING_GETADDRINFO) || \ + defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) +#if TARGET_OS_MAC && defined(__clang__) +#include +#include +#endif +#endif + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +#ifdef _WIN32 +#include + +// these are defined in wincrypt.h and it breaks compilation if BoringSSL is +// used +#undef X509_NAME +#undef X509_CERT_PAIR +#undef X509_EXTENSIONS +#undef PKCS7_SIGNER_INFO + +#ifdef _MSC_VER +#pragma comment(lib, "crypt32.lib") +#endif +#endif // _WIN32 + +#ifdef CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN +#if TARGET_OS_OSX +#include +#endif +#endif + +#include +#include +#include +#include + +#if defined(_WIN32) && defined(OPENSSL_USE_APPLINK) +#include +#endif + +#include +#include + +#if defined(OPENSSL_IS_BORINGSSL) || defined(LIBRESSL_VERSION_NUMBER) +#if OPENSSL_VERSION_NUMBER < 0x1010107f +#error Please use OpenSSL or a current version of BoringSSL +#endif +#define SSL_get1_peer_certificate SSL_get_peer_certificate +#elif OPENSSL_VERSION_NUMBER < 0x30000000L +#error Sorry, OpenSSL versions prior to 3.0.0 are not supported +#endif + +#endif // CPPHTTPLIB_OPENSSL_SUPPORT + +#ifdef CPPHTTPLIB_MBEDTLS_SUPPORT +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#ifdef _MSC_VER +#pragma comment(lib, "crypt32.lib") +#endif +#endif // _WIN32 +#ifdef CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN +#if TARGET_OS_OSX +#include +#endif +#endif + +// Mbed TLS 3.x API compatibility +#if MBEDTLS_VERSION_MAJOR >= 3 +#define CPPHTTPLIB_MBEDTLS_V3 +#endif + +#endif // CPPHTTPLIB_MBEDTLS_SUPPORT + +#ifdef CPPHTTPLIB_WOLFSSL_SUPPORT +#include + +#include + +// Fallback definitions for older wolfSSL versions (e.g., 5.6.6) +#ifndef WOLFSSL_GEN_EMAIL +#define WOLFSSL_GEN_EMAIL 1 +#endif +#ifndef WOLFSSL_GEN_DNS +#define WOLFSSL_GEN_DNS 2 +#endif +#ifndef WOLFSSL_GEN_URI +#define WOLFSSL_GEN_URI 6 +#endif +#ifndef WOLFSSL_GEN_IPADD +#define WOLFSSL_GEN_IPADD 7 +#endif + +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#ifdef _MSC_VER +#pragma comment(lib, "crypt32.lib") +#endif +#endif // _WIN32 +#ifdef CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN +#if TARGET_OS_OSX +#include +#endif +#endif +#endif // CPPHTTPLIB_WOLFSSL_SUPPORT + +// Define CPPHTTPLIB_SSL_ENABLED if any SSL backend is available +#if defined(CPPHTTPLIB_OPENSSL_SUPPORT) || \ + defined(CPPHTTPLIB_MBEDTLS_SUPPORT) || defined(CPPHTTPLIB_WOLFSSL_SUPPORT) +#define CPPHTTPLIB_SSL_ENABLED +#endif + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT +#include +#endif + +#ifdef CPPHTTPLIB_BROTLI_SUPPORT +#include +#include +#endif + +#ifdef CPPHTTPLIB_ZSTD_SUPPORT +#include +#endif + +/* + * Declaration + */ +namespace httplib { + +namespace ws { +class WebSocket; +} // namespace ws + +namespace detail { + +/* + * Backport std::make_unique from C++14. + * + * NOTE: This code came up with the following stackoverflow post: + * https://stackoverflow.com/questions/10149840/c-arrays-and-make-unique + * + */ + +template +typename std::enable_if::value, std::unique_ptr>::type +make_unique(Args &&...args) { + return std::unique_ptr(new T(std::forward(args)...)); +} + +template +typename std::enable_if::value, std::unique_ptr>::type +make_unique(std::size_t n) { + typedef typename std::remove_extent::type RT; + return std::unique_ptr(new RT[n]); +} + +namespace case_ignore { + +inline unsigned char to_lower(int c) { + const static unsigned char table[256] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, + 122, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 224, 225, 226, + 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, + 242, 243, 244, 245, 246, 215, 248, 249, 250, 251, 252, 253, 254, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, + }; + return table[(unsigned char)(char)c]; +} + +inline std::string to_lower(const std::string &s) { + std::string result = s; + std::transform( + result.begin(), result.end(), result.begin(), + [](unsigned char c) { return static_cast(to_lower(c)); }); + return result; +} + +inline bool equal(const std::string &a, const std::string &b) { + return a.size() == b.size() && + std::equal(a.begin(), a.end(), b.begin(), [](char ca, char cb) { + return to_lower(ca) == to_lower(cb); + }); +} + +struct equal_to { + bool operator()(const std::string &a, const std::string &b) const { + return equal(a, b); + } +}; + +struct hash { + size_t operator()(const std::string &key) const { + return hash_core(key.data(), key.size(), 0); + } + + size_t hash_core(const char *s, size_t l, size_t h) const { + return (l == 0) ? h + : hash_core(s + 1, l - 1, + // Unsets the 6 high bits of h, therefore no + // overflow happens + (((std::numeric_limits::max)() >> 6) & + h * 33) ^ + static_cast(to_lower(*s))); + } +}; + +template +using unordered_set = std::unordered_set; + +} // namespace case_ignore + +// This is based on +// "http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4189". + +struct scope_exit { + explicit scope_exit(std::function &&f) + : exit_function(std::move(f)), execute_on_destruction{true} {} + + scope_exit(scope_exit &&rhs) noexcept + : exit_function(std::move(rhs.exit_function)), + execute_on_destruction{rhs.execute_on_destruction} { + rhs.release(); + } + + ~scope_exit() { + if (execute_on_destruction) { this->exit_function(); } + } + + void release() { this->execute_on_destruction = false; } + +private: + scope_exit(const scope_exit &) = delete; + void operator=(const scope_exit &) = delete; + scope_exit &operator=(scope_exit &&) = delete; + + std::function exit_function; + bool execute_on_destruction; +}; + +// Simple from_chars implementation for integer and double types (C++17 +// substitute) +template struct from_chars_result { + const char *ptr; + std::errc ec; +}; + +template +inline from_chars_result from_chars(const char *first, const char *last, + T &value, int base = 10) { + value = 0; + const char *p = first; + bool negative = false; + + if (p != last && *p == '-') { + negative = true; + ++p; + } + if (p == last) { return {first, std::errc::invalid_argument}; } + + T result = 0; + for (; p != last; ++p) { + char c = *p; + int digit = -1; + if ('0' <= c && c <= '9') { + digit = c - '0'; + } else if ('a' <= c && c <= 'z') { + digit = c - 'a' + 10; + } else if ('A' <= c && c <= 'Z') { + digit = c - 'A' + 10; + } else { + break; + } + + if (digit < 0 || digit >= base) { break; } + if (result > ((std::numeric_limits::max)() - digit) / base) { + return {p, std::errc::result_out_of_range}; + } + result = result * base + digit; + } + + if (p == first || (negative && p == first + 1)) { + return {first, std::errc::invalid_argument}; + } + + value = negative ? -result : result; + return {p, std::errc{}}; +} + +// from_chars for double (hand-written, locale-independent) +// +// The only double consumed by this library is the HTTP quality value, whose +// grammar is (RFC 9110 12.4.2): +// qvalue = ( "0" [ "." 0*3DIGIT ] ) / ( "1" [ "." 0*3("0") ] ) +// i.e. a non-negative decimal with no sign, exponent, "inf"/"nan", or wide +// magnitude. So this parser recognizes exactly 1*DIGIT [ "." *DIGIT ] with +// '.' always the decimal separator (std::strtod would instead read it from the +// global C locale, mis-parsing q-values once an embedder calls +// setlocale(LC_ALL, "") into a comma-decimal locale). The caller range-checks +// the result to [0, 1], so inputs outside that range need not be distinguished +// here. Allocation-free, single pass, and free of the overflow/rounding edge +// cases that exponent and wide-range handling would introduce. +inline from_chars_result from_chars(const char *first, const char *last, + double &value) { + value = 0.0; + const char *p = first; + + // Each 1eN is exactly representable, so a single final division by the + // matching entry yields a correctly-rounded result. + static const double powers_of_ten[] = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18}; + const int max_frac_digits = + static_cast(sizeof(powers_of_ten) / sizeof(powers_of_ten[0])) - 1; + + // Accumulate digits into a 64-bit integer and remember how many were + // fractional. Two independent caps keep this bounded and safe: + // * accumulation saturates before mantissa could overflow uint64_t, and + // * frac_digits is capped at max_frac_digits so it is always a valid index + // into powers_of_ten (without this an input like "0.000...0" would never + // grow mantissa, so the saturation cap alone would not bound it). + // Both caps only drop digits far beyond the precision a q-value needs; any + // value they would change is well outside [0, 1] and rejected by the caller. + uint64_t mantissa = 0; + int frac_digits = 0; + bool seen_digit = false; + + const uint64_t limit = ((std::numeric_limits::max)() - 9) / 10; + auto accumulate = [&](char c) { + if (mantissa <= limit) { + mantissa = mantissa * 10 + static_cast(c - '0'); + return true; + } + return false; + }; + + for (; p != last && '0' <= *p && *p <= '9'; ++p) { + seen_digit = true; + accumulate(*p); + } + + if (p != last && *p == '.') { + ++p; + for (; p != last && '0' <= *p && *p <= '9'; ++p) { + seen_digit = true; + if (frac_digits < max_frac_digits && accumulate(*p)) { ++frac_digits; } + } + } + + if (!seen_digit) { return {first, std::errc::invalid_argument}; } + + value = static_cast(mantissa) / powers_of_ten[frac_digits]; + return {p, std::errc{}}; +} + +inline bool parse_port(const char *s, size_t len, int &port) { + int val = 0; + auto r = from_chars(s, s + len, val); + if (r.ec != std::errc{} || val < 1 || val > 65535) { return false; } + port = val; + return true; +} + +inline bool parse_port(const std::string &s, int &port) { + return parse_port(s.data(), s.size(), port); +} + +struct UrlComponents { + std::string scheme; + std::string host; + std::string port; + std::string path; + std::string query; +}; + +inline bool parse_url(const std::string &url, UrlComponents &uc) { + uc = {}; + size_t pos = 0; + + auto sep = url.find("://"); + if (sep != std::string::npos) { + uc.scheme = url.substr(0, sep); + + // Scheme must be [a-z]+ only + if (uc.scheme.empty()) { return false; } + for (auto c : uc.scheme) { + if (c < 'a' || c > 'z') { return false; } + } + + pos = sep + 3; + } else if (url.compare(0, 2, "//") == 0) { + pos = 2; + } + + auto has_authority_prefix = pos > 0; + auto has_authority = has_authority_prefix || (!url.empty() && url[0] != '/' && + url[0] != '?' && url[0] != '#'); + if (has_authority) { + if (pos < url.size() && url[pos] == '[') { + auto close = url.find(']', pos); + if (close == std::string::npos) { return false; } + uc.host = url.substr(pos + 1, close - pos - 1); + + // IPv6 host must be [a-fA-F0-9:]+ only + if (uc.host.empty()) { return false; } + for (auto c : uc.host) { + if (!((c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || + (c >= '0' && c <= '9') || c == ':')) { + return false; + } + } + + pos = close + 1; + } else { + auto end = url.find_first_of(":/?#", pos); + if (end == std::string::npos) { end = url.size(); } + uc.host = url.substr(pos, end - pos); + pos = end; + } + + if (pos < url.size() && url[pos] == ':') { + ++pos; + auto end = url.find_first_of("/?#", pos); + if (end == std::string::npos) { end = url.size(); } + uc.port = url.substr(pos, end - pos); + pos = end; + } + + // Without :// or //, the entire input must be consumed as host[:port]. + // If there is leftover (path, query, etc.), this is not a valid + // host[:port] string — clear and reparse as a plain path. + if (!has_authority_prefix && pos < url.size()) { + uc.host.clear(); + uc.port.clear(); + pos = 0; + } + } + + if (pos < url.size() && url[pos] != '?' && url[pos] != '#') { + auto end = url.find_first_of("?#", pos); + if (end == std::string::npos) { end = url.size(); } + uc.path = url.substr(pos, end - pos); + pos = end; + } + + if (pos < url.size() && url[pos] == '?') { + auto end = url.find('#', pos); + if (end == std::string::npos) { end = url.size(); } + uc.query = url.substr(pos, end - pos); + } + + return true; +} + +} // namespace detail + +enum class SSLVerifierResponse { + // no decision has been made, use the built-in certificate verifier + NoDecisionMade, + // connection certificate is verified and accepted + CertificateAccepted, + // connection certificate was processed but is rejected + CertificateRejected +}; + +// System CA loading policy for SSL clients. Auto (the default) loads system +// CA certs only when no custom CA is configured; enable_system_ca() switches +// to an explicit policy. +enum class SystemCAMode { Auto, Enabled, Disabled }; + +enum StatusCode { + // Information responses + Continue_100 = 100, + SwitchingProtocol_101 = 101, + Processing_102 = 102, + EarlyHints_103 = 103, + + // Successful responses + OK_200 = 200, + Created_201 = 201, + Accepted_202 = 202, + NonAuthoritativeInformation_203 = 203, + NoContent_204 = 204, + ResetContent_205 = 205, + PartialContent_206 = 206, + MultiStatus_207 = 207, + AlreadyReported_208 = 208, + IMUsed_226 = 226, + + // Redirection messages + MultipleChoices_300 = 300, + MovedPermanently_301 = 301, + Found_302 = 302, + SeeOther_303 = 303, + NotModified_304 = 304, + UseProxy_305 = 305, + unused_306 = 306, + TemporaryRedirect_307 = 307, + PermanentRedirect_308 = 308, + + // Client error responses + BadRequest_400 = 400, + Unauthorized_401 = 401, + PaymentRequired_402 = 402, + Forbidden_403 = 403, + NotFound_404 = 404, + MethodNotAllowed_405 = 405, + NotAcceptable_406 = 406, + ProxyAuthenticationRequired_407 = 407, + RequestTimeout_408 = 408, + Conflict_409 = 409, + Gone_410 = 410, + LengthRequired_411 = 411, + PreconditionFailed_412 = 412, + PayloadTooLarge_413 = 413, + UriTooLong_414 = 414, + UnsupportedMediaType_415 = 415, + RangeNotSatisfiable_416 = 416, + ExpectationFailed_417 = 417, + ImATeapot_418 = 418, + MisdirectedRequest_421 = 421, + UnprocessableContent_422 = 422, + Locked_423 = 423, + FailedDependency_424 = 424, + TooEarly_425 = 425, + UpgradeRequired_426 = 426, + PreconditionRequired_428 = 428, + TooManyRequests_429 = 429, + RequestHeaderFieldsTooLarge_431 = 431, + UnavailableForLegalReasons_451 = 451, + + // Server error responses + InternalServerError_500 = 500, + NotImplemented_501 = 501, + BadGateway_502 = 502, + ServiceUnavailable_503 = 503, + GatewayTimeout_504 = 504, + HttpVersionNotSupported_505 = 505, + VariantAlsoNegotiates_506 = 506, + InsufficientStorage_507 = 507, + LoopDetected_508 = 508, + NotExtended_510 = 510, + NetworkAuthenticationRequired_511 = 511, +}; + +using Headers = + std::unordered_multimap; + +using Params = std::multimap; +using Match = std::smatch; + +using DownloadProgress = std::function; +using UploadProgress = std::function; + +/* + * detail: type-erased storage used by UserData. + * ABI-stable regardless of C++ standard — always uses this custom + * implementation instead of std::any. + */ +namespace detail { + +using any_type_id = const void *; + +template any_type_id any_typeid() noexcept { + static const char id = 0; + return &id; +} + +struct any_storage { + virtual ~any_storage() = default; + virtual std::unique_ptr clone() const = 0; + virtual any_type_id type_id() const noexcept = 0; +}; + +template struct any_value final : any_storage { + T value; + template explicit any_value(U &&v) : value(std::forward(v)) {} + std::unique_ptr clone() const override { + return std::unique_ptr(new any_value(value)); + } + any_type_id type_id() const noexcept override { return any_typeid(); } +}; + +} // namespace detail + +class UserData { +public: + UserData() = default; + UserData(UserData &&) noexcept = default; + UserData &operator=(UserData &&) noexcept = default; + + UserData(const UserData &o) { + for (const auto &e : o.entries_) { + if (e.second) { entries_[e.first] = e.second->clone(); } + } + } + + UserData &operator=(const UserData &o) { + if (this != &o) { + entries_.clear(); + for (const auto &e : o.entries_) { + if (e.second) { entries_[e.first] = e.second->clone(); } + } + } + return *this; + } + + template void set(const std::string &key, T &&value) { + using D = typename std::decay::type; + entries_[key].reset(new detail::any_value(std::forward(value))); + } + + template T *get(const std::string &key) noexcept { + auto it = entries_.find(key); + if (it == entries_.end() || !it->second) { return nullptr; } + if (it->second->type_id() != detail::any_typeid()) { return nullptr; } + return &static_cast *>(it->second.get())->value; + } + + template const T *get(const std::string &key) const noexcept { + auto it = entries_.find(key); + if (it == entries_.end() || !it->second) { return nullptr; } + if (it->second->type_id() != detail::any_typeid()) { return nullptr; } + return &static_cast *>(it->second.get())->value; + } + + bool has(const std::string &key) const noexcept { + return entries_.find(key) != entries_.end(); + } + + void erase(const std::string &key) { entries_.erase(key); } + + void clear() noexcept { entries_.clear(); } + +private: + std::unordered_map> + entries_; +}; + +struct Response; +using ResponseHandler = std::function; + +struct FormData { + std::string name; + std::string content; + std::string filename; + std::string content_type; + Headers headers; +}; + +struct FormField { + std::string name; + std::string content; + Headers headers; +}; +using FormFields = std::multimap; + +using FormFiles = std::multimap; + +struct MultipartFormData { + FormFields fields; // Text fields from multipart + FormFiles files; // Files from multipart + + // Text field access + std::string get_field(const std::string &key, size_t id = 0) const; + std::vector get_fields(const std::string &key) const; + bool has_field(const std::string &key) const; + size_t get_field_count(const std::string &key) const; + + // File access + FormData get_file(const std::string &key, size_t id = 0) const; + std::vector get_files(const std::string &key) const; + bool has_file(const std::string &key) const; + size_t get_file_count(const std::string &key) const; +}; + +struct UploadFormData { + std::string name; + std::string content; + std::string filename; + std::string content_type; +}; +using UploadFormDataItems = std::vector; + +class DataSink { +public: + DataSink() : os(&sb_), sb_(*this) {} + + DataSink(const DataSink &) = delete; + DataSink &operator=(const DataSink &) = delete; + DataSink(DataSink &&) = delete; + DataSink &operator=(DataSink &&) = delete; + + std::function write; + std::function is_writable; + std::function done; + std::function done_with_trailer; + std::ostream os; + +private: + class data_sink_streambuf final : public std::streambuf { + public: + explicit data_sink_streambuf(DataSink &sink) : sink_(sink) {} + + protected: + std::streamsize xsputn(const char *s, std::streamsize n) override { + if (sink_.write(s, static_cast(n))) { return n; } + return 0; + } + + private: + DataSink &sink_; + }; + + data_sink_streambuf sb_; +}; + +using ContentProvider = + std::function; + +using ContentProviderWithoutLength = + std::function; + +using ContentProviderResourceReleaser = std::function; + +struct FormDataProvider { + std::string name; + ContentProviderWithoutLength provider; + std::string filename; + std::string content_type; +}; +using FormDataProviderItems = std::vector; + +inline FormDataProvider +make_file_provider(const std::string &name, const std::string &filepath, + const std::string &filename = std::string(), + const std::string &content_type = std::string()) { + FormDataProvider fdp; + fdp.name = name; + fdp.filename = filename.empty() ? filepath : filename; + fdp.content_type = content_type; + fdp.provider = [filepath](size_t offset, DataSink &sink) -> bool { + std::ifstream f(filepath, std::ios::binary); + if (!f) { return false; } + if (offset > 0) { + f.seekg(static_cast(offset)); + if (!f.good()) { + sink.done(); + return true; + } + } + char buf[8192]; + f.read(buf, sizeof(buf)); + auto n = static_cast(f.gcount()); + if (n > 0) { return sink.write(buf, n); } + sink.done(); // EOF + return true; + }; + return fdp; +} + +inline std::pair +make_file_body(const std::string &filepath) { + size_t size = 0; + { + std::ifstream f(filepath, std::ios::binary | std::ios::ate); + if (!f) { return {0, ContentProvider{}}; } + size = static_cast(f.tellg()); + } + + ContentProvider provider = [filepath](size_t offset, size_t length, + DataSink &sink) -> bool { + std::ifstream f(filepath, std::ios::binary); + if (!f) { return false; } + f.seekg(static_cast(offset)); + if (!f.good()) { return false; } + char buf[8192]; + while (length > 0) { + auto to_read = (std::min)(sizeof(buf), length); + f.read(buf, static_cast(to_read)); + auto n = static_cast(f.gcount()); + if (n == 0) { break; } + if (!sink.write(buf, n)) { return false; } + length -= n; + } + return true; + }; + return {size, std::move(provider)}; +} + +using ContentReceiverWithProgress = std::function; + +using ContentReceiver = + std::function; + +using FormDataHeader = std::function; + +class ContentReader { +public: + using Reader = std::function; + using FormDataReader = + std::function; + + ContentReader(Reader reader, FormDataReader multipart_reader) + : reader_(std::move(reader)), + formdata_reader_(std::move(multipart_reader)) {} + + bool operator()(FormDataHeader header, ContentReceiver receiver) const { + return formdata_reader_(std::move(header), std::move(receiver)); + } + + bool operator()(ContentReceiver receiver) const { + return reader_(std::move(receiver)); + } + + Reader reader_; + FormDataReader formdata_reader_; +}; + +using Range = std::pair; +using Ranges = std::vector; + +#ifdef CPPHTTPLIB_SSL_ENABLED +// TLS abstraction layer - public type definitions and API +namespace tls { + +// Opaque handles (defined as void* for abstraction) +using ctx_t = void *; +using session_t = void *; +using const_session_t = const void *; // For read-only session access +using cert_t = void *; +using ca_store_t = void *; + +// TLS versions +enum class Version { + TLS1_2 = 0x0303, + TLS1_3 = 0x0304, +}; + +// Subject Alternative Names (SAN) entry types +enum class SanType { DNS, IP, EMAIL, URI, OTHER }; + +// SAN entry structure +struct SanEntry { + SanType type; + std::string value; +}; + +// Verification context for certificate verification callback +struct VerifyContext { + session_t session; // TLS session handle + cert_t cert; // Current certificate being verified + int depth; // Certificate chain depth (0 = leaf) + bool preverify_ok; // OpenSSL/Mbed TLS pre-verification result + long error_code; // Backend-specific error code (0 = no error) + const char *error_string; // Human-readable error description + + // Certificate introspection methods + std::string subject_cn() const; + std::string issuer_name() const; + bool check_hostname(const char *hostname) const; + std::vector sans() const; + bool validity(time_t ¬_before, time_t ¬_after) const; + std::string serial() const; +}; + +using VerifyCallback = std::function; + +// TlsError codes for TLS operations (backend-independent) +enum class ErrorCode : int { + Success = 0, + WantRead, // Non-blocking: need to wait for read + WantWrite, // Non-blocking: need to wait for write + PeerClosed, // Peer closed the connection + Fatal, // Unrecoverable error + SyscallError, // System call error (check sys_errno) + CertVerifyFailed, // Certificate verification failed + HostnameMismatch, // Hostname verification failed +}; + +// TLS error information +struct TlsError { + ErrorCode code = ErrorCode::Fatal; + uint64_t backend_code = 0; // OpenSSL: ERR_get_error(), mbedTLS: return value + int sys_errno = 0; // errno when SyscallError + + // Convert verification error code to human-readable string + static std::string verify_error_to_string(long error_code); +}; + +// RAII wrapper for peer certificate +class PeerCert { +public: + PeerCert(); + PeerCert(PeerCert &&other) noexcept; + PeerCert &operator=(PeerCert &&other) noexcept; + ~PeerCert(); + + PeerCert(const PeerCert &) = delete; + PeerCert &operator=(const PeerCert &) = delete; + + explicit operator bool() const; + std::string subject_cn() const; + std::string issuer_name() const; + bool check_hostname(const char *hostname) const; + std::vector sans() const; + bool validity(time_t ¬_before, time_t ¬_after) const; + std::string serial() const; + +private: + explicit PeerCert(cert_t cert); + cert_t cert_ = nullptr; + friend PeerCert get_peer_cert_from_session(const_session_t session); +}; + +// Callback for TLS context setup (used by SSLServer constructor) +using ContextSetupCallback = std::function; + +} // namespace tls +#endif + +struct Request { + std::string method; + std::string path; + std::string matched_route; + Params params; + Headers headers; + Headers trailers; + std::string body; + + std::string remote_addr; + int remote_port = -1; + std::string local_addr; + int local_port = -1; + + // for server + std::string version; + std::string target; + MultipartFormData form; + Ranges ranges; + Match matches; + std::unordered_map path_params; + std::function is_connection_closed = []() { return true; }; + + // for client + std::vector accept_content_types; + ResponseHandler response_handler; + ContentReceiverWithProgress content_receiver; + DownloadProgress download_progress; + UploadProgress upload_progress; + + bool has_header(const std::string &key) const; + std::string get_header_value(const std::string &key, const char *def = "", + size_t id = 0) const; + size_t get_header_value_u64(const std::string &key, size_t def = 0, + size_t id = 0) const; + size_t get_header_value_count(const std::string &key) const; + void set_header(const std::string &key, const std::string &val); + + bool has_trailer(const std::string &key) const; + std::string get_trailer_value(const std::string &key, size_t id = 0) const; + size_t get_trailer_value_count(const std::string &key) const; + + bool has_param(const std::string &key) const; + std::string get_param_value(const std::string &key, size_t id = 0) const; + std::vector get_param_values(const std::string &key) const; + size_t get_param_value_count(const std::string &key) const; + + bool is_multipart_form_data() const; + + // private members... + bool body_consumed_ = false; + size_t redirect_count_ = CPPHTTPLIB_REDIRECT_MAX_COUNT; + size_t content_length_ = 0; + ContentProvider content_provider_; + bool is_chunked_content_provider_ = false; + size_t authorization_count_ = 0; + std::chrono::time_point start_time_ = + (std::chrono::steady_clock::time_point::min)(); + +#ifdef CPPHTTPLIB_SSL_ENABLED + tls::const_session_t ssl = nullptr; + tls::PeerCert peer_cert() const; + std::string sni() const; +#endif +}; + +struct Response { + std::string version; + int status = -1; + std::string reason; + Headers headers; + Headers trailers; + std::string body; + std::string location; // Redirect location + + // User-defined context — set by pre-routing/pre-request handlers and read + // by route handlers to pass arbitrary data (e.g. decoded auth tokens). + UserData user_data; + + bool has_header(const std::string &key) const; + std::string get_header_value(const std::string &key, const char *def = "", + size_t id = 0) const; + size_t get_header_value_u64(const std::string &key, size_t def = 0, + size_t id = 0) const; + size_t get_header_value_count(const std::string &key) const; + void set_header(const std::string &key, const std::string &val); + + bool has_trailer(const std::string &key) const; + std::string get_trailer_value(const std::string &key, size_t id = 0) const; + size_t get_trailer_value_count(const std::string &key) const; + + void set_redirect(const std::string &url, int status = StatusCode::Found_302); + void set_content(const char *s, size_t n, const std::string &content_type); + void set_content(const std::string &s, const std::string &content_type); + void set_content(std::string &&s, const std::string &content_type); + + void set_content_provider( + size_t length, const std::string &content_type, ContentProvider provider, + ContentProviderResourceReleaser resource_releaser = nullptr); + + void set_content_provider( + const std::string &content_type, ContentProviderWithoutLength provider, + ContentProviderResourceReleaser resource_releaser = nullptr); + + void set_chunked_content_provider( + const std::string &content_type, ContentProviderWithoutLength provider, + ContentProviderResourceReleaser resource_releaser = nullptr); + + void set_file_content(const std::string &path, + const std::string &content_type); + void set_file_content(const std::string &path); + + Response() = default; + Response(const Response &) = default; + Response &operator=(const Response &) = default; + Response(Response &&) = default; + Response &operator=(Response &&) = default; + ~Response() { + if (content_provider_resource_releaser_) { + content_provider_resource_releaser_(content_provider_success_); + } + } + + // private members... + size_t content_length_ = 0; + ContentProvider content_provider_; + ContentProviderResourceReleaser content_provider_resource_releaser_; + bool is_chunked_content_provider_ = false; + bool content_provider_success_ = false; + std::string file_content_path_; + std::string file_content_content_type_; +}; + +enum class Error { + Success = 0, + Unknown, + Connection, + BindIPAddress, + Read, + Write, + ExceedRedirectCount, + Canceled, + SSLConnection, + SSLLoadingCerts, + SSLServerVerification, + SSLServerHostnameVerification, + UnsupportedMultipartBoundaryChars, + Compression, + ConnectionTimeout, + ProxyConnection, + ConnectionClosed, + Timeout, + ResourceExhaustion, + TooManyFormDataFiles, + ExceedMaxPayloadSize, + ExceedUriMaxLength, + ExceedMaxSocketDescriptorCount, + InvalidRequestLine, + InvalidHTTPMethod, + InvalidHTTPVersion, + InvalidHeaders, + MultipartParsing, + OpenFile, + Listen, + GetSockName, + UnsupportedAddressFamily, + HTTPParsing, + InvalidRangeHeader, + + // For internal use only + SSLPeerCouldBeClosed_, +}; + +std::string to_string(Error error); + +std::ostream &operator<<(std::ostream &os, const Error &obj); + +class Stream { +public: + virtual ~Stream() = default; + + virtual bool is_readable() const = 0; + virtual bool wait_readable() const = 0; + virtual bool wait_writable() const = 0; + virtual bool is_peer_alive() const { return wait_writable(); } + + virtual ssize_t read(char *ptr, size_t size) = 0; + virtual ssize_t write(const char *ptr, size_t size) = 0; + virtual void get_remote_ip_and_port(std::string &ip, int &port) const = 0; + virtual void get_local_ip_and_port(std::string &ip, int &port) const = 0; + virtual socket_t socket() const = 0; + + virtual time_t duration() const = 0; + + virtual void set_read_timeout(time_t sec, time_t usec = 0) { + (void)sec; + (void)usec; + } + + ssize_t write(const char *ptr); + ssize_t write(const std::string &s); + + Error get_error() const { return error_; } + +protected: + Error error_ = Error::Success; +}; + +class TaskQueue { +public: + TaskQueue() = default; + virtual ~TaskQueue() = default; + + virtual bool enqueue(std::function fn) = 0; + virtual void shutdown() = 0; + + virtual void on_idle() {} +}; + +class ThreadPool final : public TaskQueue { +public: + explicit ThreadPool(size_t n, size_t max_n = 0, size_t mqr = 0); + ThreadPool(const ThreadPool &) = delete; + ~ThreadPool() override = default; + + bool enqueue(std::function fn) override; + void shutdown() override; + +private: + void worker(bool is_dynamic); + void move_to_finished(std::thread::id id); + void cleanup_finished_threads(); + + size_t base_thread_count_; + size_t max_thread_count_; + size_t max_queued_requests_; + size_t idle_thread_count_; + + bool shutdown_; + + std::list> jobs_; + std::vector threads_; // base threads + std::list dynamic_threads_; // dynamic threads + std::vector + finished_threads_; // exited dynamic threads awaiting join + + std::condition_variable cond_; + std::mutex mutex_; +}; + +using Logger = std::function; + +// Forward declaration for Error type +enum class Error; +using ErrorLogger = std::function; + +using SocketOptions = std::function; + +void default_socket_options(socket_t sock); + +bool set_socket_opt(socket_t sock, int level, int optname, int optval); + +const char *status_message(int status); + +std::string to_string(Error error); + +std::ostream &operator<<(std::ostream &os, const Error &obj); + +std::string get_bearer_token_auth(const Request &req); + +namespace detail { + +class MatcherBase { +public: + MatcherBase(std::string pattern) : pattern_(std::move(pattern)) {} + virtual ~MatcherBase() = default; + + const std::string &pattern() const { return pattern_; } + + // Match request path and populate its matches and + virtual bool match(Request &request) const = 0; + +private: + std::string pattern_; +}; + +/** + * Captures parameters in request path and stores them in Request::path_params + * + * Capture name is a substring of a pattern from : to /. + * The rest of the pattern is matched against the request path directly + * Parameters are captured starting from the next character after + * the end of the last matched static pattern fragment until the next /. + * + * Example pattern: + * "/path/fragments/:capture/more/fragments/:second_capture" + * Static fragments: + * "/path/fragments/", "more/fragments/" + * + * Given the following request path: + * "/path/fragments/:1/more/fragments/:2" + * the resulting capture will be + * {{"capture", "1"}, {"second_capture", "2"}} + */ +class PathParamsMatcher final : public MatcherBase { +public: + PathParamsMatcher(const std::string &pattern); + + bool match(Request &request) const override; + +private: + // Treat segment separators as the end of path parameter capture + // Does not need to handle query parameters as they are parsed before path + // matching + static constexpr char separator = '/'; + + // Contains static path fragments to match against, excluding the '/' after + // path params + // Fragments are separated by path params + std::vector static_fragments_; + // Stores the names of the path parameters to be used as keys in the + // Request::path_params map + std::vector param_names_; +}; + +/** + * Performs std::regex_match on request path + * and stores the result in Request::matches + * + * Note that regex match is performed directly on the whole request. + * This means that wildcard patterns may match multiple path segments with /: + * "/begin/(.*)/end" will match both "/begin/middle/end" and "/begin/1/2/end". + */ +class RegexMatcher final : public MatcherBase { +public: + RegexMatcher(const std::string &pattern) + : MatcherBase(pattern), regex_(pattern) {} + + bool match(Request &request) const override; + +private: + std::regex regex_; +}; + +int close_socket(socket_t sock) noexcept; + +ssize_t write_headers(Stream &strm, const Headers &headers); + +bool set_socket_opt_time(socket_t sock, int level, int optname, time_t sec, + time_t usec); + +size_t get_multipart_content_length(const UploadFormDataItems &items, + const std::string &boundary); + +ContentProvider +make_multipart_content_provider(const UploadFormDataItems &items, + const std::string &boundary); + +} // namespace detail + +class Server { +public: + using Handler = std::function; + + using ExceptionHandler = + std::function; + + enum class HandlerResponse { + Handled, + Unhandled, + }; + using HandlerWithResponse = + std::function; + + using HandlerWithContentReader = std::function; + + using Expect100ContinueHandler = + std::function; + + using StartHandler = std::function; + + using WebSocketHandler = + std::function; + using SubProtocolSelector = + std::function &protocols)>; + + Server(); + + virtual ~Server(); + + virtual bool is_valid() const; + + Server &Get(const std::string &pattern, Handler handler); + Server &Post(const std::string &pattern, Handler handler); + Server &Post(const std::string &pattern, HandlerWithContentReader handler); + Server &Put(const std::string &pattern, Handler handler); + Server &Put(const std::string &pattern, HandlerWithContentReader handler); + Server &Patch(const std::string &pattern, Handler handler); + Server &Patch(const std::string &pattern, HandlerWithContentReader handler); + Server &Delete(const std::string &pattern, Handler handler); + Server &Delete(const std::string &pattern, HandlerWithContentReader handler); + Server &Options(const std::string &pattern, Handler handler); + + Server &WebSocket(const std::string &pattern, WebSocketHandler handler); + Server &WebSocket(const std::string &pattern, WebSocketHandler handler, + SubProtocolSelector sub_protocol_selector); + + bool set_base_dir(const std::string &dir, + const std::string &mount_point = std::string()); + bool set_mount_point(const std::string &mount_point, const std::string &dir, + Headers headers = Headers()); + bool remove_mount_point(const std::string &mount_point); + Server &set_file_extension_and_mimetype_mapping(const std::string &ext, + const std::string &mime); + Server &set_default_file_mimetype(const std::string &mime); + Server &set_file_request_handler(Handler handler); + + template + Server &set_error_handler(ErrorHandlerFunc &&handler) { + return set_error_handler_core( + std::forward(handler), + std::is_convertible{}); + } + + Server &set_exception_handler(ExceptionHandler handler); + + Server &set_pre_routing_handler(HandlerWithResponse handler); + Server &set_post_routing_handler(Handler handler); + + Server &set_pre_request_handler(HandlerWithResponse handler); + + Server &set_expect_100_continue_handler(Expect100ContinueHandler handler); + + Server &set_start_handler(StartHandler handler); + + Server &set_logger(Logger logger); + Server &set_pre_compression_logger(Logger logger); + Server &set_error_logger(ErrorLogger error_logger); + + Server &set_address_family(int family); + Server &set_tcp_nodelay(bool on); + Server &set_ipv6_v6only(bool on); + Server &set_socket_options(SocketOptions socket_options); + + Server &set_default_headers(Headers headers); + Server & + set_header_writer(std::function const &writer); + + Server &set_trusted_proxies(const std::vector &proxies); + + Server &set_keep_alive_max_count(size_t count); + Server &set_keep_alive_timeout(time_t sec); + template + Server & + set_keep_alive_timeout(const std::chrono::duration &duration); + + Server &set_read_timeout(time_t sec, time_t usec = 0); + template + Server &set_read_timeout(const std::chrono::duration &duration); + + Server &set_write_timeout(time_t sec, time_t usec = 0); + template + Server &set_write_timeout(const std::chrono::duration &duration); + + Server &set_idle_interval(time_t sec, time_t usec = 0); + template + Server &set_idle_interval(const std::chrono::duration &duration); + + Server &set_payload_max_length(size_t length); + + Server &set_websocket_ping_interval(time_t sec); + template + Server &set_websocket_ping_interval( + const std::chrono::duration &duration); + + Server &set_websocket_max_missed_pongs(int count); + + bool bind_to_port(const std::string &host, int port, int socket_flags = 0); + int bind_to_any_port(const std::string &host, int socket_flags = 0); + bool listen_after_bind(); + + bool listen(const std::string &host, int port, int socket_flags = 0); + + bool is_running() const; + void wait_until_ready() const; + void stop() noexcept; + void decommission(); + + std::function new_task_queue; + +protected: + bool process_request(Stream &strm, const std::string &remote_addr, + int remote_port, const std::string &local_addr, + int local_port, bool close_connection, + bool &connection_closed, + const std::function &setup_request, + bool *websocket_upgraded = nullptr); + + std::atomic svr_sock_{INVALID_SOCKET}; + + std::vector trusted_proxies_; + + size_t keep_alive_max_count_ = CPPHTTPLIB_KEEPALIVE_MAX_COUNT; + time_t keep_alive_timeout_sec_ = CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND; + time_t read_timeout_sec_ = CPPHTTPLIB_SERVER_READ_TIMEOUT_SECOND; + time_t read_timeout_usec_ = CPPHTTPLIB_SERVER_READ_TIMEOUT_USECOND; + time_t write_timeout_sec_ = CPPHTTPLIB_SERVER_WRITE_TIMEOUT_SECOND; + time_t write_timeout_usec_ = CPPHTTPLIB_SERVER_WRITE_TIMEOUT_USECOND; + time_t idle_interval_sec_ = CPPHTTPLIB_IDLE_INTERVAL_SECOND; + time_t idle_interval_usec_ = CPPHTTPLIB_IDLE_INTERVAL_USECOND; + size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH; + time_t websocket_ping_interval_sec_ = + CPPHTTPLIB_WEBSOCKET_PING_INTERVAL_SECOND; + int websocket_max_missed_pongs_ = CPPHTTPLIB_WEBSOCKET_MAX_MISSED_PONGS; + +private: + using Handlers = + std::vector, Handler>>; + using HandlersForContentReader = + std::vector, + HandlerWithContentReader>>; + + static std::unique_ptr + make_matcher(const std::string &pattern); + + template + Server &add_handler( + std::vector, H>> &handlers, + const std::string &pattern, H handler) { + handlers.emplace_back(make_matcher(pattern), std::move(handler)); + return *this; + } + + Server &set_error_handler_core(HandlerWithResponse handler, std::true_type); + Server &set_error_handler_core(Handler handler, std::false_type); + + socket_t create_server_socket(const std::string &host, int port, + int socket_flags, + SocketOptions socket_options) const; + int bind_internal(const std::string &host, int port, int socket_flags); + bool listen_internal(); + + bool routing(Request &req, Response &res, Stream &strm); + bool handle_file_request(Request &req, Response &res); + bool check_if_not_modified(const Request &req, Response &res, + const std::string &etag, time_t mtime) const; + bool check_if_range(Request &req, const std::string &etag, + time_t mtime) const; + bool dispatch_request(Request &req, Response &res, const Handlers &handlers, + Stream &strm); + bool dispatch_request_for_content_reader( + Request &req, Response &res, ContentReader content_reader, + const HandlersForContentReader &handlers) const; + + bool parse_request_line(const char *s, Request &req) const; + void apply_ranges(const Request &req, Response &res, + std::string &content_type, std::string &boundary) const; + bool write_response(Stream &strm, bool close_connection, Request &req, + Response &res); + bool write_response_with_content(Stream &strm, bool close_connection, + const Request &req, Response &res); + bool write_response_core(Stream &strm, bool close_connection, + const Request &req, Response &res, + bool need_apply_ranges); + bool write_content_with_provider(Stream &strm, const Request &req, + Response &res, const std::string &boundary, + const std::string &content_type); + bool read_content(Stream &strm, Request &req, Response &res); + bool read_content_with_content_receiver(Stream &strm, Request &req, + Response &res, + ContentReceiver receiver, + FormDataHeader multipart_header, + ContentReceiver multipart_receiver); + bool read_content_core(Stream &strm, Request &req, Response &res, + ContentReceiver receiver, + FormDataHeader multipart_header, + ContentReceiver multipart_receiver) const; + + virtual bool process_and_close_socket(socket_t sock); + + void output_log(const Request &req, const Response &res) const; + void output_pre_compression_log(const Request &req, + const Response &res) const; + void output_error_log(const Error &err, const Request *req) const; + + std::atomic is_running_{false}; + std::atomic is_decommissioned{false}; + + struct MountPointEntry { + std::string mount_point; + std::string base_dir; + std::string resolved_base_dir; + Headers headers; + }; + std::vector base_dirs_; + std::map file_extension_and_mimetype_map_; + std::string default_file_mimetype_ = "application/octet-stream"; + Handler file_request_handler_; + + Handlers get_handlers_; + Handlers post_handlers_; + HandlersForContentReader post_handlers_for_content_reader_; + Handlers put_handlers_; + HandlersForContentReader put_handlers_for_content_reader_; + Handlers patch_handlers_; + HandlersForContentReader patch_handlers_for_content_reader_; + Handlers delete_handlers_; + HandlersForContentReader delete_handlers_for_content_reader_; + Handlers options_handlers_; + + struct WebSocketHandlerEntry { + std::unique_ptr matcher; + WebSocketHandler handler; + SubProtocolSelector sub_protocol_selector; + }; + using WebSocketHandlers = std::vector; + WebSocketHandlers websocket_handlers_; + + HandlerWithResponse error_handler_; + ExceptionHandler exception_handler_; + HandlerWithResponse pre_routing_handler_; + Handler post_routing_handler_; + HandlerWithResponse pre_request_handler_; + Expect100ContinueHandler expect_100_continue_handler_; + StartHandler start_handler_; + + mutable std::mutex logger_mutex_; + Logger logger_; + Logger pre_compression_logger_; + ErrorLogger error_logger_; + + int address_family_ = AF_UNSPEC; + bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY; + bool ipv6_v6only_ = CPPHTTPLIB_IPV6_V6ONLY; + SocketOptions socket_options_ = default_socket_options; + + Headers default_headers_; + std::function header_writer_ = + detail::write_headers; +}; + +class Result { +public: + Result() = default; + Result(std::unique_ptr &&res, Error err, + Headers &&request_headers = Headers{}) + : res_(std::move(res)), err_(err), + request_headers_(std::move(request_headers)) {} + // Response + operator bool() const { return res_ != nullptr; } + bool operator==(std::nullptr_t) const { return res_ == nullptr; } + bool operator!=(std::nullptr_t) const { return res_ != nullptr; } + const Response &value() const { return *res_; } + Response &value() { return *res_; } + const Response &operator*() const { return *res_; } + Response &operator*() { return *res_; } + const Response *operator->() const { return res_.get(); } + Response *operator->() { return res_.get(); } + + // Error + Error error() const { return err_; } + + // Request Headers + bool has_request_header(const std::string &key) const; + std::string get_request_header_value(const std::string &key, + const char *def = "", + size_t id = 0) const; + size_t get_request_header_value_u64(const std::string &key, size_t def = 0, + size_t id = 0) const; + size_t get_request_header_value_count(const std::string &key) const; + +private: + std::unique_ptr res_; + Error err_ = Error::Unknown; + Headers request_headers_; + +#ifdef CPPHTTPLIB_SSL_ENABLED +public: + Result(std::unique_ptr &&res, Error err, Headers &&request_headers, + int ssl_error) + : res_(std::move(res)), err_(err), + request_headers_(std::move(request_headers)), ssl_error_(ssl_error) {} + Result(std::unique_ptr &&res, Error err, Headers &&request_headers, + int ssl_error, uint64_t ssl_backend_error) + : res_(std::move(res)), err_(err), + request_headers_(std::move(request_headers)), ssl_error_(ssl_error), + ssl_backend_error_(ssl_backend_error) {} + + int ssl_error() const { return ssl_error_; } + uint64_t ssl_backend_error() const { return ssl_backend_error_; } + +private: + int ssl_error_ = 0; + uint64_t ssl_backend_error_ = 0; +#endif +}; + +struct ClientConnection { + socket_t sock = INVALID_SOCKET; + + bool is_open() const { return sock != INVALID_SOCKET; } + + ClientConnection() = default; + + ~ClientConnection(); + + ClientConnection(const ClientConnection &) = delete; + ClientConnection &operator=(const ClientConnection &) = delete; + + ClientConnection(ClientConnection &&other) noexcept + : sock(other.sock) +#ifdef CPPHTTPLIB_SSL_ENABLED + , + session(other.session) +#endif + { + other.sock = INVALID_SOCKET; +#ifdef CPPHTTPLIB_SSL_ENABLED + other.session = nullptr; +#endif + } + + ClientConnection &operator=(ClientConnection &&other) noexcept { + if (this != &other) { + sock = other.sock; + other.sock = INVALID_SOCKET; +#ifdef CPPHTTPLIB_SSL_ENABLED + session = other.session; + other.session = nullptr; +#endif + } + return *this; + } + +#ifdef CPPHTTPLIB_SSL_ENABLED + tls::session_t session = nullptr; +#endif +}; + +namespace detail { + +struct ChunkedDecoder; + +struct BodyReader { + Stream *stream = nullptr; + bool has_content_length = false; + size_t content_length = 0; + size_t payload_max_length = CPPHTTPLIB_PAYLOAD_MAX_LENGTH; + size_t bytes_read = 0; + bool chunked = false; + bool eof = false; + std::unique_ptr chunked_decoder; + Error last_error = Error::Success; + + ssize_t read(char *buf, size_t len); + bool has_error() const { return last_error != Error::Success; } +}; + +inline ssize_t read_body_content(Stream *stream, BodyReader &br, char *buf, + size_t len) { + (void)stream; + return br.read(buf, len); +} + +class decompressor; + +enum class NoProxyKind { + Wildcard, // "*" + HostnameSuffix, // "example.com" or ".example.com" + IPv4Cidr, // "10.0.0.0/8" (or single IP, treated as /32) + IPv6Cidr, // "fe80::/10" (or single IP, treated as /128) +}; + +// Unified 16-byte buffer holding either a v4 (first 4 bytes) or v6 address. +// Lets one CIDR matcher cover both families. +using IPBytes = std::array; + +struct NoProxyEntry { + NoProxyKind kind = NoProxyKind::Wildcard; + std::string hostname_pattern; // lowercased, leading/trailing dot stripped + IPBytes net{}; + int prefix_bits = 0; +}; + +struct NormalizedTarget { + std::string hostname; // lowercase; brackets and trailing dot removed + bool is_ipv4 = false; + bool is_ipv6 = false; + IPBytes ip{}; +}; + +} // namespace detail + +class ClientImpl { +public: + explicit ClientImpl(const std::string &host); + + explicit ClientImpl(const std::string &host, int port); + + explicit ClientImpl(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path); + + virtual ~ClientImpl(); + + virtual bool is_valid() const; + + struct StreamHandle { + std::unique_ptr response; + Error error = Error::Success; + + StreamHandle() = default; + StreamHandle(const StreamHandle &) = delete; + StreamHandle &operator=(const StreamHandle &) = delete; + StreamHandle(StreamHandle &&) = default; + StreamHandle &operator=(StreamHandle &&) = default; + ~StreamHandle() = default; + + bool is_valid() const { + return response != nullptr && error == Error::Success; + } + + ssize_t read(char *buf, size_t len); + void parse_trailers_if_needed(); + Error get_read_error() const { return body_reader_.last_error; } + bool has_read_error() const { return body_reader_.has_error(); } + + bool trailers_parsed_ = false; + + private: + friend class ClientImpl; + + ssize_t read_with_decompression(char *buf, size_t len); + + std::unique_ptr connection_; + std::unique_ptr socket_stream_; + Stream *stream_ = nullptr; + detail::BodyReader body_reader_; + + std::unique_ptr decompressor_; + std::string decompress_buffer_; + size_t decompress_offset_ = 0; + size_t decompressed_bytes_read_ = 0; + }; + + // clang-format off + Result Get(const std::string &path, DownloadProgress progress = nullptr); + Result Get(const std::string &path, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Head(const std::string &path); + Result Head(const std::string &path, const Headers &headers); + + Result Post(const std::string &path); + Result Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Post(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Params ¶ms); + Result Post(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers); + Result Post(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const Params ¶ms); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Put(const std::string &path); + Result Put(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Put(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Params ¶ms); + Result Put(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers); + Result Put(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const Params ¶ms); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Patch(const std::string &path); + Result Patch(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Patch(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Params ¶ms); + Result Patch(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const Params ¶ms); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Delete(const std::string &path, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const char *body, size_t content_length, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const std::string &body, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Params ¶ms, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const Params ¶ms, DownloadProgress progress = nullptr); + + Result Options(const std::string &path); + Result Options(const std::string &path, const Headers &headers); + // clang-format on + + // Streaming API: Open a stream for reading response body incrementally + // Socket ownership is transferred to StreamHandle for true streaming + // Supports all HTTP methods (GET, POST, PUT, PATCH, DELETE, etc.) + StreamHandle open_stream(const std::string &method, const std::string &path, + const Params ¶ms = {}, + const Headers &headers = {}, + const std::string &body = {}, + const std::string &content_type = {}); + + bool send(Request &req, Response &res, Error &error); + Result send(const Request &req); + + void stop(); + + std::string host() const; + int port() const; + + size_t is_socket_open() const; + socket_t socket() const; + + void set_hostname_addr_map(std::map addr_map); + + void set_default_headers(Headers headers); + + void + set_header_writer(std::function const &writer); + + void set_address_family(int family); + void set_tcp_nodelay(bool on); + void set_ipv6_v6only(bool on); + void set_socket_options(SocketOptions socket_options); + + void set_connection_timeout(time_t sec, time_t usec = 0); + template + void + set_connection_timeout(const std::chrono::duration &duration); + + void set_read_timeout(time_t sec, time_t usec = 0); + template + void set_read_timeout(const std::chrono::duration &duration); + + void set_write_timeout(time_t sec, time_t usec = 0); + template + void set_write_timeout(const std::chrono::duration &duration); + + void set_max_timeout(time_t msec); + template + void set_max_timeout(const std::chrono::duration &duration); + + void set_basic_auth(const std::string &username, const std::string &password); + void set_bearer_token_auth(const std::string &token); + + void set_keep_alive(bool on); + void set_follow_location(bool on); + + void set_path_encode(bool on); + + void set_compress(bool on); + + void set_decompress(bool on); + + void set_payload_max_length(size_t length); + + void set_interface(const std::string &intf); + + void set_proxy(const std::string &host, int port); + void set_proxy_basic_auth(const std::string &username, + const std::string &password); + void set_proxy_bearer_token_auth(const std::string &token); + void set_no_proxy(const std::vector &patterns); + + void set_logger(Logger logger); + void set_error_logger(ErrorLogger error_logger); + +protected: + struct Socket { + socket_t sock = INVALID_SOCKET; + + // For Mbed TLS compatibility: start_time for request timeout tracking + std::chrono::time_point start_time_; + + bool is_open() const { return sock != INVALID_SOCKET; } + +#ifdef CPPHTTPLIB_SSL_ENABLED + tls::session_t ssl = nullptr; +#endif + }; + + virtual bool create_and_connect_socket(Socket &socket, Error &error); + virtual bool ensure_socket_connection(Socket &socket, Error &error); + virtual bool setup_proxy_connection( + Socket &socket, + std::chrono::time_point start_time, + Response &res, bool &success, Error &error); + + bool is_proxy_enabled_for_host(const std::string &host) const; + + // All of: + // shutdown_ssl + // shutdown_socket + // close_socket + // disconnect + // should ONLY be called when socket_mutex_ is locked, and only when + // no other thread is using the socket. + virtual void shutdown_ssl(Socket &socket, bool shutdown_gracefully); + void shutdown_socket(Socket &socket) const; + void close_socket(Socket &socket); + void disconnect(bool gracefully); + + bool process_request(Stream &strm, Request &req, Response &res, + bool close_connection, Error &error); + + bool write_content_with_provider(Stream &strm, const Request &req, + Error &error) const; + + void copy_settings(const ClientImpl &rhs); + + void output_log(const Request &req, const Response &res) const; + void output_error_log(const Error &err, const Request *req) const; + + // Socket endpoint information + const std::string host_; + const int port_; + + // Current open socket + Socket socket_; + mutable std::mutex socket_mutex_; + std::recursive_mutex request_mutex_; + + // These are all protected under socket_mutex + size_t socket_requests_in_flight_ = 0; + std::thread::id socket_requests_are_from_thread_ = std::thread::id(); + bool socket_should_be_closed_when_request_is_done_ = false; + + // Hostname-IP map + std::map addr_map_; + + // Default headers + Headers default_headers_; + + // Header writer + std::function header_writer_ = + detail::write_headers; + + // Settings + std::string client_cert_path_; + std::string client_key_path_; + + time_t connection_timeout_sec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND; + time_t connection_timeout_usec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND; + time_t read_timeout_sec_ = CPPHTTPLIB_CLIENT_READ_TIMEOUT_SECOND; + time_t read_timeout_usec_ = CPPHTTPLIB_CLIENT_READ_TIMEOUT_USECOND; + time_t write_timeout_sec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND; + time_t write_timeout_usec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND; + time_t max_timeout_msec_ = CPPHTTPLIB_CLIENT_MAX_TIMEOUT_MSECOND; + + std::string basic_auth_username_; + std::string basic_auth_password_; + std::string bearer_token_auth_token_; + + bool keep_alive_ = false; + bool follow_location_ = false; + + bool path_encode_ = true; + + int address_family_ = AF_UNSPEC; + bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY; + bool ipv6_v6only_ = CPPHTTPLIB_IPV6_V6ONLY; + SocketOptions socket_options_ = nullptr; + + bool compress_ = false; + bool decompress_ = true; + + size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH; + bool has_payload_max_length_ = false; + + std::string interface_; + + std::string proxy_host_; + int proxy_port_ = -1; + + std::string proxy_basic_auth_username_; + std::string proxy_basic_auth_password_; + std::string proxy_bearer_token_auth_token_; + + std::vector no_proxy_entries_; + + mutable detail::NormalizedTarget host_normalized_; + mutable bool host_normalized_valid_ = false; + + mutable std::mutex logger_mutex_; + Logger logger_; + ErrorLogger error_logger_; + +private: + bool send_(Request &req, Response &res, Error &error); + Result send_(Request &&req); + + socket_t create_client_socket(Error &error) const; + bool read_response_line(Stream &strm, const Request &req, Response &res, + bool skip_100_continue = true) const; + bool write_request(Stream &strm, Request &req, bool close_connection, + Error &error, bool skip_body = false); + bool write_request_body(Stream &strm, Request &req, Error &error); + void prepare_default_headers(Request &r, bool for_stream, + const std::string &ct); + bool redirect(Request &req, Response &res, Error &error); + bool create_redirect_client(const std::string &scheme, + const std::string &host, int port, Request &req, + Response &res, const std::string &path, + const std::string &location, Error &error); + template void setup_redirect_client(ClientType &client); + bool handle_request(Stream &strm, Request &req, Response &res, + bool close_connection, Error &error); + std::unique_ptr send_with_content_provider_and_receiver( + Request &req, const char *body, size_t content_length, + ContentProvider content_provider, + ContentProviderWithoutLength content_provider_without_length, + const std::string &content_type, ContentReceiver content_receiver, + Error &error); + Result send_with_content_provider_and_receiver( + const std::string &method, const std::string &path, + const Headers &headers, const char *body, size_t content_length, + ContentProvider content_provider, + ContentProviderWithoutLength content_provider_without_length, + const std::string &content_type, ContentReceiver content_receiver, + UploadProgress progress); + ContentProviderWithoutLength get_multipart_content_provider( + const std::string &boundary, const UploadFormDataItems &items, + const FormDataProviderItems &provider_items) const; + + virtual bool + process_socket(const Socket &socket, + std::chrono::time_point start_time, + std::function callback); + virtual bool is_ssl() const; + + void transfer_socket_ownership_to_handle(StreamHandle &handle); + +#ifdef CPPHTTPLIB_SSL_ENABLED +public: + void set_digest_auth(const std::string &username, + const std::string &password); + void set_proxy_digest_auth(const std::string &username, + const std::string &password); + void set_ca_cert_path(const std::string &ca_cert_file_path, + const std::string &ca_cert_dir_path = std::string()); + void enable_server_certificate_verification(bool enabled); + void enable_server_hostname_verification(bool enabled); + void enable_system_ca(bool enabled); + +protected: + std::string digest_auth_username_; + std::string digest_auth_password_; + std::string proxy_digest_auth_username_; + std::string proxy_digest_auth_password_; + std::string ca_cert_file_path_; + std::string ca_cert_dir_path_; + bool server_certificate_verification_ = true; + bool server_hostname_verification_ = true; + SystemCAMode system_ca_mode_ = SystemCAMode::Auto; + std::string ca_cert_pem_; // Store CA cert PEM for redirect transfer + int last_ssl_error_ = 0; + uint64_t last_backend_error_ = 0; +#endif +}; + +class Client { +public: + // Universal interface + explicit Client(const std::string &scheme_host_port); + + explicit Client(const std::string &scheme_host_port, + const std::string &client_cert_path, + const std::string &client_key_path); + + // HTTP only interface + explicit Client(const std::string &host, int port); + + explicit Client(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path); + + Client(Client &&) = default; + Client &operator=(Client &&) = default; + + ~Client(); + + bool is_valid() const; + + // clang-format off + Result Get(const std::string &path, DownloadProgress progress = nullptr); + Result Get(const std::string &path, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Head(const std::string &path); + Result Head(const std::string &path, const Headers &headers); + + Result Post(const std::string &path); + Result Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Post(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Params ¶ms); + Result Post(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers); + Result Post(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const Params ¶ms); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Put(const std::string &path); + Result Put(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Put(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Params ¶ms); + Result Put(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers); + Result Put(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const Params ¶ms); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Patch(const std::string &path); + Result Patch(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Patch(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Params ¶ms); + Result Patch(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers); + Result Patch(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, ContentReceiver content_receiver, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const Params ¶ms); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Delete(const std::string &path, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const char *body, size_t content_length, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const std::string &body, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Params ¶ms, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const Params ¶ms, DownloadProgress progress = nullptr); + + Result Options(const std::string &path); + Result Options(const std::string &path, const Headers &headers); + // clang-format on + + // Streaming API: Open a stream for reading response body incrementally + // Socket ownership is transferred to StreamHandle for true streaming + // Supports all HTTP methods (GET, POST, PUT, PATCH, DELETE, etc.) + ClientImpl::StreamHandle open_stream(const std::string &method, + const std::string &path, + const Params ¶ms = {}, + const Headers &headers = {}, + const std::string &body = {}, + const std::string &content_type = {}); + + bool send(Request &req, Response &res, Error &error); + Result send(const Request &req); + + void stop(); + + std::string host() const; + int port() const; + + size_t is_socket_open() const; + socket_t socket() const; + + void set_hostname_addr_map(std::map addr_map); + + void set_default_headers(Headers headers); + + void + set_header_writer(std::function const &writer); + + void set_address_family(int family); + void set_tcp_nodelay(bool on); + void set_socket_options(SocketOptions socket_options); + + void set_connection_timeout(time_t sec, time_t usec = 0); + template + void + set_connection_timeout(const std::chrono::duration &duration); + + void set_read_timeout(time_t sec, time_t usec = 0); + template + void set_read_timeout(const std::chrono::duration &duration); + + void set_write_timeout(time_t sec, time_t usec = 0); + template + void set_write_timeout(const std::chrono::duration &duration); + + void set_max_timeout(time_t msec); + template + void set_max_timeout(const std::chrono::duration &duration); + + void set_basic_auth(const std::string &username, const std::string &password); + void set_bearer_token_auth(const std::string &token); + + void set_keep_alive(bool on); + void set_follow_location(bool on); + + void set_path_encode(bool on); + + void set_compress(bool on); + + void set_decompress(bool on); + + void set_payload_max_length(size_t length); + + void set_interface(const std::string &intf); + + void set_proxy(const std::string &host, int port); + void set_proxy_basic_auth(const std::string &username, + const std::string &password); + void set_proxy_bearer_token_auth(const std::string &token); + void set_no_proxy(const std::vector &patterns); + void set_logger(Logger logger); + void set_error_logger(ErrorLogger error_logger); + +private: + std::unique_ptr cli_; + +#ifdef CPPHTTPLIB_SSL_ENABLED +public: + void set_digest_auth(const std::string &username, + const std::string &password); + void set_proxy_digest_auth(const std::string &username, + const std::string &password); + void enable_server_certificate_verification(bool enabled); + void enable_server_hostname_verification(bool enabled); + void enable_system_ca(bool enabled); + void set_ca_cert_path(const std::string &ca_cert_file_path, + const std::string &ca_cert_dir_path = std::string()); + + void set_ca_cert_store(tls::ca_store_t ca_cert_store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); + + void set_server_certificate_verifier(tls::VerifyCallback verifier); + + void set_session_verifier( + std::function verifier); + + tls::ctx_t tls_context() const; + +#ifdef CPPHTTPLIB_WINDOWS_AUTOMATIC_ROOT_CERTIFICATES_UPDATE + void enable_windows_certificate_verification(bool enabled); +#endif + +private: + bool is_ssl_ = false; +#endif +}; + +#ifdef CPPHTTPLIB_SSL_ENABLED +class SSLServer : public Server { +public: + SSLServer(const char *cert_path, const char *private_key_path, + const char *client_ca_cert_file_path = nullptr, + const char *client_ca_cert_dir_path = nullptr, + const char *private_key_password = nullptr); + + struct PemMemory { + const char *cert_pem; + size_t cert_pem_len; + const char *key_pem; + size_t key_pem_len; + const char *client_ca_pem; + size_t client_ca_pem_len; + const char *private_key_password; + }; + explicit SSLServer(const PemMemory &pem); + + // The callback receives the ctx_t handle which can be cast to the + // appropriate backend type (SSL_CTX* for OpenSSL, + // tls::impl::MbedTlsContext* for Mbed TLS) + explicit SSLServer(const tls::ContextSetupCallback &setup_callback); + + ~SSLServer() override; + + bool is_valid() const override; + + bool update_certs_pem(const char *cert_pem, const char *key_pem, + const char *client_ca_pem = nullptr, + const char *password = nullptr); + + tls::ctx_t tls_context() const { return ctx_; } + + int ssl_last_error() const { return last_ssl_error_; } + +private: + bool process_and_close_socket(socket_t sock) override; + + tls::ctx_t ctx_ = nullptr; + std::mutex ctx_mutex_; + + int last_ssl_error_ = 0; +}; + +class SSLClient final : public ClientImpl { +public: + explicit SSLClient(const std::string &host); + + explicit SSLClient(const std::string &host, int port); + + explicit SSLClient(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path, + const std::string &private_key_password = std::string()); + + struct PemMemory { + const char *cert_pem; + size_t cert_pem_len; + const char *key_pem; + size_t key_pem_len; + const char *private_key_password; + }; + explicit SSLClient(const std::string &host, int port, const PemMemory &pem); + + ~SSLClient() override; + + bool is_valid() const override; + + void set_ca_cert_store(tls::ca_store_t ca_cert_store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); + + void set_server_certificate_verifier(tls::VerifyCallback verifier); + + // Post-handshake session verifier (backend-independent) + void set_session_verifier( + std::function verifier); + + tls::ctx_t tls_context() const { return ctx_; } + +#ifdef CPPHTTPLIB_WINDOWS_AUTOMATIC_ROOT_CERTIFICATES_UPDATE + void enable_windows_certificate_verification(bool enabled); +#endif + +private: + bool create_and_connect_socket(Socket &socket, Error &error) override; + bool ensure_socket_connection(Socket &socket, Error &error) override; + void shutdown_ssl(Socket &socket, bool shutdown_gracefully) override; + void shutdown_ssl_impl(Socket &socket, bool shutdown_gracefully); + + bool + process_socket(const Socket &socket, + std::chrono::time_point start_time, + std::function callback) override; + bool is_ssl() const override; + + bool setup_proxy_connection( + Socket &socket, + std::chrono::time_point start_time, + Response &res, bool &success, Error &error) override; + bool connect_with_proxy( + Socket &sock, + std::chrono::time_point start_time, + Response &res, bool &success, Error &error); + bool initialize_ssl(Socket &socket, Error &error); + + void init_ctx(); + void reset_ctx_on_error(); + + bool load_certs(); + + tls::ctx_t ctx_ = nullptr; + std::mutex ctx_mutex_; + std::once_flag initialize_cert_; + + // Tracks whether a custom CA store was applied via set_ca_cert_store(), + // since the store handle itself is owned by ctx_ and leaves no other trace. + // Used to keep custom CA configuration exclusive with system CA loading. + bool ca_cert_store_set_ = false; + + long verify_result_ = 0; + + std::function session_verifier_; + +#ifdef CPPHTTPLIB_WINDOWS_AUTOMATIC_ROOT_CERTIFICATES_UPDATE + bool enable_windows_cert_verification_ = true; +#endif + + friend class ClientImpl; +}; +#endif // CPPHTTPLIB_SSL_ENABLED + +namespace detail { + +template +inline void duration_to_sec_and_usec(const T &duration, U callback) { + auto sec = std::chrono::duration_cast(duration).count(); + auto usec = std::chrono::duration_cast( + duration - std::chrono::seconds(sec)) + .count(); + callback(static_cast(sec), static_cast(usec)); +} + +template inline constexpr size_t str_len(const char (&)[N]) { + return N - 1; +} + +inline bool is_numeric(const std::string &str) { + return !str.empty() && + std::all_of(str.cbegin(), str.cend(), + [](unsigned char c) { return std::isdigit(c); }); +} + +inline size_t get_header_value_u64(const Headers &headers, + const std::string &key, size_t def, + size_t id, bool &is_invalid_value) { + is_invalid_value = false; + auto rng = headers.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { + if (is_numeric(it->second)) { + return static_cast(std::strtoull(it->second.data(), nullptr, 10)); + } else { + is_invalid_value = true; + } + } + return def; +} + +inline size_t get_header_value_u64(const Headers &headers, + const std::string &key, size_t def, + size_t id) { + auto dummy = false; + return get_header_value_u64(headers, key, def, id, dummy); +} + +} // namespace detail + +template +inline Server & +Server::set_read_timeout(const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); }); + return *this; +} + +template +inline Server & +Server::set_write_timeout(const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_write_timeout(sec, usec); }); + return *this; +} + +template +inline Server & +Server::set_idle_interval(const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_idle_interval(sec, usec); }); + return *this; +} + +template +inline void ClientImpl::set_connection_timeout( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec(duration, [&](time_t sec, time_t usec) { + set_connection_timeout(sec, usec); + }); +} + +template +inline void ClientImpl::set_read_timeout( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); }); +} + +template +inline void ClientImpl::set_write_timeout( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_write_timeout(sec, usec); }); +} + +template +inline void ClientImpl::set_max_timeout( + const std::chrono::duration &duration) { + auto msec = + std::chrono::duration_cast(duration).count(); + set_max_timeout(msec); +} + +template +inline void Client::set_connection_timeout( + const std::chrono::duration &duration) { + cli_->set_connection_timeout(duration); +} + +template +inline void +Client::set_read_timeout(const std::chrono::duration &duration) { + cli_->set_read_timeout(duration); +} + +template +inline void +Client::set_write_timeout(const std::chrono::duration &duration) { + cli_->set_write_timeout(duration); +} + +inline void Client::set_max_timeout(time_t msec) { + cli_->set_max_timeout(msec); +} + +template +inline void +Client::set_max_timeout(const std::chrono::duration &duration) { + cli_->set_max_timeout(duration); +} + +/* + * Forward declarations and types that will be part of the .h file if split into + * .h + .cc. + */ + +std::string hosted_at(const std::string &hostname); + +void hosted_at(const std::string &hostname, std::vector &addrs); + +// JavaScript-style URL encoding/decoding functions +std::string encode_uri_component(const std::string &value); +std::string encode_uri(const std::string &value); +std::string decode_uri_component(const std::string &value); +std::string decode_uri(const std::string &value); + +// RFC 3986 compliant URL component encoding/decoding functions +std::string encode_path_component(const std::string &component); +std::string decode_path_component(const std::string &component); +std::string encode_query_component(const std::string &component, + bool space_as_plus = true); +std::string decode_query_component(const std::string &component, + bool plus_as_space = true); + +std::string sanitize_filename(const std::string &filename); + +std::string append_query_params(const std::string &path, const Params ¶ms); + +std::pair make_range_header(const Ranges &ranges); + +std::pair +make_basic_authentication_header(const std::string &username, + const std::string &password, + bool is_proxy = false); + +namespace detail { + +#if defined(_WIN32) +inline std::wstring u8string_to_wstring(const char *s) { + if (!s) { return std::wstring(); } + + auto len = static_cast(strlen(s)); + if (!len) { return std::wstring(); } + + auto wlen = ::MultiByteToWideChar(CP_UTF8, 0, s, len, nullptr, 0); + if (!wlen) { return std::wstring(); } + + std::wstring ws; + ws.resize(wlen); + wlen = ::MultiByteToWideChar( + CP_UTF8, 0, s, len, + const_cast(reinterpret_cast(ws.data())), wlen); + if (wlen != static_cast(ws.size())) { ws.clear(); } + return ws; +} +#endif + +struct FileStat { + FileStat(const std::string &path); + bool is_file() const; + bool is_dir() const; + time_t mtime() const; + size_t size() const; + +private: +#if defined(_WIN32) + struct _stat st_; +#else + struct stat st_; +#endif + int ret_ = -1; +}; + +std::string make_host_and_port_string(const std::string &host, int port, + bool is_ssl); + +std::string trim_copy(const std::string &s); + +void divide( + const char *data, std::size_t size, char d, + std::function + fn); + +void divide( + const std::string &str, char d, + std::function + fn); + +void split(const char *b, const char *e, char d, + std::function fn); + +void split(const char *b, const char *e, char d, size_t m, + std::function fn); + +bool process_client_socket( + socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, time_t write_timeout_usec, + time_t max_timeout_msec, + std::chrono::time_point start_time, + std::function callback); + +socket_t create_client_socket(const std::string &host, const std::string &ip, + int port, int address_family, bool tcp_nodelay, + bool ipv6_v6only, SocketOptions socket_options, + time_t connection_timeout_sec, + time_t connection_timeout_usec, + time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, + time_t write_timeout_usec, + const std::string &intf, Error &error); + +const char *get_header_value(const Headers &headers, const std::string &key, + const char *def, size_t id); + +std::string params_to_query_str(const Params ¶ms); + +void parse_query_text(const char *data, std::size_t size, Params ¶ms); + +void parse_query_text(const std::string &s, Params ¶ms); + +bool parse_multipart_boundary(const std::string &content_type, + std::string &boundary); + +bool parse_range_header(const std::string &s, Ranges &ranges); + +bool parse_accept_header(const std::string &s, + std::vector &content_types); + +ssize_t send_socket(socket_t sock, const void *ptr, size_t size, int flags); + +ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags); + +enum class EncodingType { None = 0, Gzip, Brotli, Zstd }; + +EncodingType encoding_type(const Request &req, const Response &res); + +class BufferStream final : public Stream { +public: + BufferStream() = default; + ~BufferStream() override = default; + + bool is_readable() const override; + bool wait_readable() const override; + bool wait_writable() const override; + ssize_t read(char *ptr, size_t size) override; + ssize_t write(const char *ptr, size_t size) override; + void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; + socket_t socket() const override; + time_t duration() const override; + + const std::string &get_buffer() const; + +private: + std::string buffer; + size_t position = 0; +}; + +class compressor { +public: + virtual ~compressor() = default; + + typedef std::function Callback; + virtual bool compress(const char *data, size_t data_length, bool last, + Callback callback) = 0; +}; + +class decompressor { +public: + virtual ~decompressor() = default; + + virtual bool is_valid() const = 0; + + typedef std::function Callback; + virtual bool decompress(const char *data, size_t data_length, + Callback callback) = 0; +}; + +class nocompressor final : public compressor { +public: + ~nocompressor() override = default; + + bool compress(const char *data, size_t data_length, bool /*last*/, + Callback callback) override; +}; + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT +class gzip_compressor final : public compressor { +public: + gzip_compressor(); + ~gzip_compressor() override; + + bool compress(const char *data, size_t data_length, bool last, + Callback callback) override; + +private: + bool is_valid_ = false; + z_stream strm_; +}; + +class gzip_decompressor final : public decompressor { +public: + gzip_decompressor(); + ~gzip_decompressor() override; + + bool is_valid() const override; + + bool decompress(const char *data, size_t data_length, + Callback callback) override; + +private: + bool is_valid_ = false; + z_stream strm_; +}; +#endif + +#ifdef CPPHTTPLIB_BROTLI_SUPPORT +class brotli_compressor final : public compressor { +public: + brotli_compressor(); + ~brotli_compressor(); + + bool compress(const char *data, size_t data_length, bool last, + Callback callback) override; + +private: + BrotliEncoderState *state_ = nullptr; +}; + +class brotli_decompressor final : public decompressor { +public: + brotli_decompressor(); + ~brotli_decompressor(); + + bool is_valid() const override; + + bool decompress(const char *data, size_t data_length, + Callback callback) override; + +private: + BrotliDecoderResult decoder_r; + BrotliDecoderState *decoder_s = nullptr; +}; +#endif + +#ifdef CPPHTTPLIB_ZSTD_SUPPORT +class zstd_compressor : public compressor { +public: + zstd_compressor(); + ~zstd_compressor(); + + bool compress(const char *data, size_t data_length, bool last, + Callback callback) override; + +private: + ZSTD_CCtx *ctx_ = nullptr; +}; + +class zstd_decompressor : public decompressor { +public: + zstd_decompressor(); + ~zstd_decompressor(); + + bool is_valid() const override; + + bool decompress(const char *data, size_t data_length, + Callback callback) override; + +private: + ZSTD_DCtx *ctx_ = nullptr; +}; +#endif + +// NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer` +// to store data. The call can set memory on stack for performance. +class stream_line_reader { +public: + stream_line_reader(Stream &strm, char *fixed_buffer, + size_t fixed_buffer_size); + const char *ptr() const; + size_t size() const; + bool end_with_crlf() const; + bool getline(); + +private: + void append(char c); + + Stream &strm_; + char *fixed_buffer_; + const size_t fixed_buffer_size_; + size_t fixed_buffer_used_size_ = 0; + std::string growable_buffer_; +}; + +bool parse_trailers(stream_line_reader &line_reader, Headers &dest, + const Headers &src_headers); + +struct ChunkedDecoder { + Stream &strm; + size_t chunk_remaining = 0; + bool finished = false; + char line_buf[64]; + size_t last_chunk_total = 0; + size_t last_chunk_offset = 0; + + explicit ChunkedDecoder(Stream &s); + + ssize_t read_payload(char *buf, size_t len, size_t &out_chunk_offset, + size_t &out_chunk_total); + + bool parse_trailers_into(Headers &dest, const Headers &src_headers); +}; + +class mmap { +public: + mmap(const char *path); + ~mmap(); + + bool open(const char *path); + void close(); + + bool is_open() const; + size_t size() const; + const char *data() const; + +private: +#if defined(_WIN32) + HANDLE hFile_ = NULL; + HANDLE hMapping_ = NULL; +#else + int fd_ = -1; +#endif + size_t size_ = 0; + void *addr_ = nullptr; + bool is_open_empty_file = false; +}; + +// NOTE: https://www.rfc-editor.org/rfc/rfc9110#section-5 +namespace fields { + +bool is_token_char(char c); +bool is_token(const std::string &s); +bool is_field_name(const std::string &s); +bool is_vchar(char c); +bool is_obs_text(char c); +bool is_field_vchar(char c); +bool is_field_content(const std::string &s); +bool is_field_value(const std::string &s); + +} // namespace fields +} // namespace detail + +/* + * TLS Abstraction Layer Declarations + */ + +#ifdef CPPHTTPLIB_SSL_ENABLED +// TLS abstraction layer - backend-specific type declarations +#ifdef CPPHTTPLIB_MBEDTLS_SUPPORT +namespace tls { +namespace impl { + +// Mbed TLS context wrapper (holds config, entropy, DRBG, CA chain, own +// cert/key). This struct is accessible via tls::impl for use in SSL context +// setup callbacks (cast ctx_t to tls::impl::MbedTlsContext*). +struct MbedTlsContext { + mbedtls_ssl_config conf; + mbedtls_entropy_context entropy; + mbedtls_ctr_drbg_context ctr_drbg; + mbedtls_x509_crt ca_chain; + mbedtls_x509_crt own_cert; + mbedtls_pk_context own_key; + bool is_server = false; + bool verify_client = false; + bool has_verify_callback = false; + + MbedTlsContext(); + ~MbedTlsContext(); + + MbedTlsContext(const MbedTlsContext &) = delete; + MbedTlsContext &operator=(const MbedTlsContext &) = delete; +}; + +} // namespace impl +} // namespace tls +#endif + +#ifdef CPPHTTPLIB_WOLFSSL_SUPPORT +namespace tls { +namespace impl { + +// wolfSSL context wrapper (holds WOLFSSL_CTX and related state). +// This struct is accessible via tls::impl for use in SSL context +// setup callbacks (cast ctx_t to tls::impl::WolfSSLContext*). +struct WolfSSLContext { + WOLFSSL_CTX *ctx = nullptr; + bool is_server = false; + bool verify_client = false; + bool has_verify_callback = false; + std::string ca_pem_data_; // accumulated PEM for get_ca_names/get_ca_certs + + WolfSSLContext(); + ~WolfSSLContext(); + + WolfSSLContext(const WolfSSLContext &) = delete; + WolfSSLContext &operator=(const WolfSSLContext &) = delete; +}; + +// CA store for wolfSSL: holds raw PEM bytes to allow reloading into any ctx +struct WolfSSLCAStore { + std::string pem_data; +}; + +} // namespace impl +} // namespace tls +#endif + +#endif // CPPHTTPLIB_SSL_ENABLED + +namespace stream { + +class Result { +public: + Result(); + explicit Result(ClientImpl::StreamHandle &&handle, size_t chunk_size = 8192); + Result(Result &&other) noexcept; + Result &operator=(Result &&other) noexcept; + Result(const Result &) = delete; + Result &operator=(const Result &) = delete; + + // Response info + bool is_valid() const; + explicit operator bool() const; + int status() const; + const Headers &headers() const; + std::string get_header_value(const std::string &key, + const char *def = "") const; + bool has_header(const std::string &key) const; + Error error() const; + Error read_error() const; + bool has_read_error() const; + + // Stream reading + bool next(); + const char *data() const; + size_t size() const; + std::string read_all(); + +private: + ClientImpl::StreamHandle handle_; + std::string buffer_; + size_t current_size_ = 0; + size_t chunk_size_; + bool finished_ = false; +}; + +// GET +template +inline Result Get(ClientType &cli, const std::string &path, + size_t chunk_size = 8192) { + return Result{cli.open_stream("GET", path), chunk_size}; +} + +template +inline Result Get(ClientType &cli, const std::string &path, + const Headers &headers, size_t chunk_size = 8192) { + return Result{cli.open_stream("GET", path, {}, headers), chunk_size}; +} + +template +inline Result Get(ClientType &cli, const std::string &path, + const Params ¶ms, size_t chunk_size = 8192) { + return Result{cli.open_stream("GET", path, params), chunk_size}; +} + +template +inline Result Get(ClientType &cli, const std::string &path, + const Params ¶ms, const Headers &headers, + size_t chunk_size = 8192) { + return Result{cli.open_stream("GET", path, params, headers), chunk_size}; +} + +// POST +template +inline Result Post(ClientType &cli, const std::string &path, + const std::string &body, const std::string &content_type, + size_t chunk_size = 8192) { + return Result{cli.open_stream("POST", path, {}, {}, body, content_type), + chunk_size}; +} + +template +inline Result Post(ClientType &cli, const std::string &path, + const Headers &headers, const std::string &body, + const std::string &content_type, size_t chunk_size = 8192) { + return Result{cli.open_stream("POST", path, {}, headers, body, content_type), + chunk_size}; +} + +template +inline Result Post(ClientType &cli, const std::string &path, + const Params ¶ms, const std::string &body, + const std::string &content_type, size_t chunk_size = 8192) { + return Result{cli.open_stream("POST", path, params, {}, body, content_type), + chunk_size}; +} + +template +inline Result Post(ClientType &cli, const std::string &path, + const Params ¶ms, const Headers &headers, + const std::string &body, const std::string &content_type, + size_t chunk_size = 8192) { + return Result{ + cli.open_stream("POST", path, params, headers, body, content_type), + chunk_size}; +} + +// PUT +template +inline Result Put(ClientType &cli, const std::string &path, + const std::string &body, const std::string &content_type, + size_t chunk_size = 8192) { + return Result{cli.open_stream("PUT", path, {}, {}, body, content_type), + chunk_size}; +} + +template +inline Result Put(ClientType &cli, const std::string &path, + const Headers &headers, const std::string &body, + const std::string &content_type, size_t chunk_size = 8192) { + return Result{cli.open_stream("PUT", path, {}, headers, body, content_type), + chunk_size}; +} + +template +inline Result Put(ClientType &cli, const std::string &path, + const Params ¶ms, const std::string &body, + const std::string &content_type, size_t chunk_size = 8192) { + return Result{cli.open_stream("PUT", path, params, {}, body, content_type), + chunk_size}; +} + +template +inline Result Put(ClientType &cli, const std::string &path, + const Params ¶ms, const Headers &headers, + const std::string &body, const std::string &content_type, + size_t chunk_size = 8192) { + return Result{ + cli.open_stream("PUT", path, params, headers, body, content_type), + chunk_size}; +} + +// PATCH +template +inline Result Patch(ClientType &cli, const std::string &path, + const std::string &body, const std::string &content_type, + size_t chunk_size = 8192) { + return Result{cli.open_stream("PATCH", path, {}, {}, body, content_type), + chunk_size}; +} + +template +inline Result Patch(ClientType &cli, const std::string &path, + const Headers &headers, const std::string &body, + const std::string &content_type, size_t chunk_size = 8192) { + return Result{cli.open_stream("PATCH", path, {}, headers, body, content_type), + chunk_size}; +} + +template +inline Result Patch(ClientType &cli, const std::string &path, + const Params ¶ms, const std::string &body, + const std::string &content_type, size_t chunk_size = 8192) { + return Result{cli.open_stream("PATCH", path, params, {}, body, content_type), + chunk_size}; +} + +template +inline Result Patch(ClientType &cli, const std::string &path, + const Params ¶ms, const Headers &headers, + const std::string &body, const std::string &content_type, + size_t chunk_size = 8192) { + return Result{ + cli.open_stream("PATCH", path, params, headers, body, content_type), + chunk_size}; +} + +// DELETE +template +inline Result Delete(ClientType &cli, const std::string &path, + size_t chunk_size = 8192) { + return Result{cli.open_stream("DELETE", path), chunk_size}; +} + +template +inline Result Delete(ClientType &cli, const std::string &path, + const Headers &headers, size_t chunk_size = 8192) { + return Result{cli.open_stream("DELETE", path, {}, headers), chunk_size}; +} + +template +inline Result Delete(ClientType &cli, const std::string &path, + const std::string &body, const std::string &content_type, + size_t chunk_size = 8192) { + return Result{cli.open_stream("DELETE", path, {}, {}, body, content_type), + chunk_size}; +} + +template +inline Result Delete(ClientType &cli, const std::string &path, + const Headers &headers, const std::string &body, + const std::string &content_type, + size_t chunk_size = 8192) { + return Result{ + cli.open_stream("DELETE", path, {}, headers, body, content_type), + chunk_size}; +} + +template +inline Result Delete(ClientType &cli, const std::string &path, + const Params ¶ms, size_t chunk_size = 8192) { + return Result{cli.open_stream("DELETE", path, params), chunk_size}; +} + +template +inline Result Delete(ClientType &cli, const std::string &path, + const Params ¶ms, const Headers &headers, + size_t chunk_size = 8192) { + return Result{cli.open_stream("DELETE", path, params, headers), chunk_size}; +} + +template +inline Result Delete(ClientType &cli, const std::string &path, + const Params ¶ms, const std::string &body, + const std::string &content_type, + size_t chunk_size = 8192) { + return Result{cli.open_stream("DELETE", path, params, {}, body, content_type), + chunk_size}; +} + +template +inline Result Delete(ClientType &cli, const std::string &path, + const Params ¶ms, const Headers &headers, + const std::string &body, const std::string &content_type, + size_t chunk_size = 8192) { + return Result{ + cli.open_stream("DELETE", path, params, headers, body, content_type), + chunk_size}; +} + +// HEAD +template +inline Result Head(ClientType &cli, const std::string &path, + size_t chunk_size = 8192) { + return Result{cli.open_stream("HEAD", path), chunk_size}; +} + +template +inline Result Head(ClientType &cli, const std::string &path, + const Headers &headers, size_t chunk_size = 8192) { + return Result{cli.open_stream("HEAD", path, {}, headers), chunk_size}; +} + +template +inline Result Head(ClientType &cli, const std::string &path, + const Params ¶ms, size_t chunk_size = 8192) { + return Result{cli.open_stream("HEAD", path, params), chunk_size}; +} + +template +inline Result Head(ClientType &cli, const std::string &path, + const Params ¶ms, const Headers &headers, + size_t chunk_size = 8192) { + return Result{cli.open_stream("HEAD", path, params, headers), chunk_size}; +} + +// OPTIONS +template +inline Result Options(ClientType &cli, const std::string &path, + size_t chunk_size = 8192) { + return Result{cli.open_stream("OPTIONS", path), chunk_size}; +} + +template +inline Result Options(ClientType &cli, const std::string &path, + const Headers &headers, size_t chunk_size = 8192) { + return Result{cli.open_stream("OPTIONS", path, {}, headers), chunk_size}; +} + +template +inline Result Options(ClientType &cli, const std::string &path, + const Params ¶ms, size_t chunk_size = 8192) { + return Result{cli.open_stream("OPTIONS", path, params), chunk_size}; +} + +template +inline Result Options(ClientType &cli, const std::string &path, + const Params ¶ms, const Headers &headers, + size_t chunk_size = 8192) { + return Result{cli.open_stream("OPTIONS", path, params, headers), chunk_size}; +} + +} // namespace stream + +namespace sse { + +struct SSEMessage { + std::string event; // Event type (default: "message") + std::string data; // Event payload + std::string id; // Event ID for Last-Event-ID header + + SSEMessage(); + void clear(); +}; + +class SSEClient { +public: + using MessageHandler = std::function; + using ErrorHandler = std::function; + using OpenHandler = std::function; + + SSEClient(Client &client, const std::string &path); + SSEClient(Client &client, const std::string &path, const Headers &headers); + ~SSEClient(); + + SSEClient(const SSEClient &) = delete; + SSEClient &operator=(const SSEClient &) = delete; + + // Event handlers + SSEClient &on_message(MessageHandler handler); + SSEClient &on_event(const std::string &type, MessageHandler handler); + SSEClient &on_open(OpenHandler handler); + SSEClient &on_error(ErrorHandler handler); + SSEClient &set_reconnect_interval(int ms); + SSEClient &set_max_reconnect_attempts(int n); + + // Update headers (thread-safe) + SSEClient &set_headers(const Headers &headers); + + // State accessors + bool is_connected() const; + const std::string &last_event_id() const; + + // Blocking start - runs event loop with auto-reconnect + void start(); + + // Non-blocking start - runs in background thread + void start_async(); + + // Stop the client (thread-safe) + void stop(); + +private: + bool parse_sse_line(const std::string &line, SSEMessage &msg, int &retry_ms); + void run_event_loop(); + void dispatch_event(const SSEMessage &msg); + bool should_reconnect(int count) const; + void wait_for_reconnect(); + + // Client and path + Client &client_; + std::string path_; + Headers headers_; + mutable std::mutex headers_mutex_; + + // Callbacks + MessageHandler on_message_; + std::map event_handlers_; + OpenHandler on_open_; + ErrorHandler on_error_; + + // Configuration + int reconnect_interval_ms_ = 3000; + int max_reconnect_attempts_ = 0; // 0 = unlimited + + // State + std::atomic running_{false}; + std::atomic connected_{false}; + std::string last_event_id_; + + // Async support + std::thread async_thread_; +}; + +} // namespace sse + +namespace ws { + +enum class Opcode : uint8_t { + Continuation = 0x0, + Text = 0x1, + Binary = 0x2, + Close = 0x8, + Ping = 0x9, + Pong = 0xA, +}; + +enum class CloseStatus : uint16_t { + Normal = 1000, + GoingAway = 1001, + ProtocolError = 1002, + UnsupportedData = 1003, + NoStatus = 1005, + Abnormal = 1006, + InvalidPayload = 1007, + PolicyViolation = 1008, + MessageTooBig = 1009, + MandatoryExtension = 1010, + InternalError = 1011, +}; + +enum ReadResult : int { Fail = 0, Text = 1, Binary = 2 }; + +class WebSocket { +public: + WebSocket(const WebSocket &) = delete; + WebSocket &operator=(const WebSocket &) = delete; + ~WebSocket(); + + ReadResult read(std::string &msg); + bool send(const std::string &data); + bool send(const char *data, size_t len); + void close(CloseStatus status = CloseStatus::Normal, + const std::string &reason = ""); + const Request &request() const; + bool is_open() const; + +private: + friend class httplib::Server; + friend class WebSocketClient; + + WebSocket( + Stream &strm, const Request &req, bool is_server, + time_t ping_interval_sec = CPPHTTPLIB_WEBSOCKET_PING_INTERVAL_SECOND, + int max_missed_pongs = CPPHTTPLIB_WEBSOCKET_MAX_MISSED_PONGS) + : strm_(strm), req_(req), is_server_(is_server), + ping_interval_sec_(ping_interval_sec), + max_missed_pongs_(max_missed_pongs) { + start_heartbeat(); + } + + WebSocket( + std::unique_ptr &&owned_strm, const Request &req, bool is_server, + time_t ping_interval_sec = CPPHTTPLIB_WEBSOCKET_PING_INTERVAL_SECOND, + int max_missed_pongs = CPPHTTPLIB_WEBSOCKET_MAX_MISSED_PONGS) + : strm_(*owned_strm), owned_strm_(std::move(owned_strm)), req_(req), + is_server_(is_server), ping_interval_sec_(ping_interval_sec), + max_missed_pongs_(max_missed_pongs) { + start_heartbeat(); + } + + void start_heartbeat(); + bool send_frame(Opcode op, const char *data, size_t len, bool fin = true); + + Stream &strm_; + std::unique_ptr owned_strm_; + Request req_; + bool is_server_; + time_t ping_interval_sec_; + int max_missed_pongs_; + int unacked_pings_ = 0; + std::atomic closed_{false}; + std::mutex write_mutex_; + std::thread ping_thread_; + std::mutex ping_mutex_; + std::condition_variable ping_cv_; +}; + +class WebSocketClient { +public: + explicit WebSocketClient(const std::string &scheme_host_port_path, + const Headers &headers = {}); + + ~WebSocketClient(); + WebSocketClient(const WebSocketClient &) = delete; + WebSocketClient &operator=(const WebSocketClient &) = delete; + + bool is_valid() const; + + bool connect(); + ReadResult read(std::string &msg); + bool send(const std::string &data); + bool send(const char *data, size_t len); + void close(CloseStatus status = CloseStatus::Normal, + const std::string &reason = ""); + bool is_open() const; + const std::string &subprotocol() const; + void set_read_timeout(time_t sec, time_t usec = 0); + void set_write_timeout(time_t sec, time_t usec = 0); + void set_websocket_ping_interval(time_t sec); + void set_websocket_max_missed_pongs(int count); + void set_tcp_nodelay(bool on); + void set_address_family(int family); + void set_ipv6_v6only(bool on); + void set_socket_options(SocketOptions socket_options); + void set_connection_timeout(time_t sec, time_t usec = 0); + void set_interface(const std::string &intf); + void set_hostname_addr_map(std::map addr_map); + +#ifdef CPPHTTPLIB_SSL_ENABLED + void set_ca_cert_path(const std::string &path); + void set_ca_cert_store(tls::ca_store_t store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); + void enable_server_certificate_verification(bool enabled); + void enable_system_ca(bool enabled); +#endif + +private: + void shutdown_and_close(); + bool create_stream(std::unique_ptr &strm); + + std::string host_; + int port_; + std::string path_; + Headers headers_; + std::string subprotocol_; + bool is_valid_ = false; + socket_t sock_ = INVALID_SOCKET; + std::unique_ptr ws_; + time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND; + time_t read_timeout_usec_ = 0; + time_t write_timeout_sec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND; + time_t write_timeout_usec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND; + time_t websocket_ping_interval_sec_ = + CPPHTTPLIB_WEBSOCKET_PING_INTERVAL_SECOND; + int websocket_max_missed_pongs_ = CPPHTTPLIB_WEBSOCKET_MAX_MISSED_PONGS; + int address_family_ = AF_UNSPEC; + bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY; + bool ipv6_v6only_ = CPPHTTPLIB_IPV6_V6ONLY; + SocketOptions socket_options_ = nullptr; + time_t connection_timeout_sec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND; + time_t connection_timeout_usec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND; + std::string interface_; + + // Hostname-IP map + std::map addr_map_; + +#ifdef CPPHTTPLIB_SSL_ENABLED + bool is_ssl_ = false; + tls::ctx_t tls_ctx_ = nullptr; + tls::session_t tls_session_ = nullptr; + std::string ca_cert_file_path_; + bool custom_ca_loaded_ = false; + bool certs_loaded_ = false; + SystemCAMode system_ca_mode_ = SystemCAMode::Auto; + bool server_certificate_verification_ = true; +#endif +}; + +namespace impl { + +bool is_valid_utf8(const std::string &s); + +bool read_websocket_frame(Stream &strm, Opcode &opcode, std::string &payload, + bool &fin, bool expect_masked, size_t max_len); + +} // namespace impl + +} // namespace ws + +// ---------------------------------------------------------------------------- + +/* + * Implementation that will be part of the .cc file if split into .h + .cc. + */ + +namespace stream { + +// stream::Result implementations +inline Result::Result() : chunk_size_(8192) {} + +inline Result::Result(ClientImpl::StreamHandle &&handle, size_t chunk_size) + : handle_(std::move(handle)), chunk_size_(chunk_size) {} + +inline Result::Result(Result &&other) noexcept + : handle_(std::move(other.handle_)), buffer_(std::move(other.buffer_)), + current_size_(other.current_size_), chunk_size_(other.chunk_size_), + finished_(other.finished_) { + other.current_size_ = 0; + other.finished_ = true; +} + +inline Result &Result::operator=(Result &&other) noexcept { + if (this != &other) { + handle_ = std::move(other.handle_); + buffer_ = std::move(other.buffer_); + current_size_ = other.current_size_; + chunk_size_ = other.chunk_size_; + finished_ = other.finished_; + other.current_size_ = 0; + other.finished_ = true; + } + return *this; +} + +inline bool Result::is_valid() const { return handle_.is_valid(); } +inline Result::operator bool() const { return is_valid(); } + +inline int Result::status() const { + return handle_.response ? handle_.response->status : -1; +} + +inline const Headers &Result::headers() const { + static const Headers empty_headers; + return handle_.response ? handle_.response->headers : empty_headers; +} + +inline std::string Result::get_header_value(const std::string &key, + const char *def) const { + return handle_.response ? handle_.response->get_header_value(key, def) : def; +} + +inline bool Result::has_header(const std::string &key) const { + return handle_.response ? handle_.response->has_header(key) : false; +} + +inline Error Result::error() const { return handle_.error; } +inline Error Result::read_error() const { return handle_.get_read_error(); } +inline bool Result::has_read_error() const { return handle_.has_read_error(); } + +inline bool Result::next() { + if (!handle_.is_valid() || finished_) { return false; } + + if (buffer_.size() < chunk_size_) { buffer_.resize(chunk_size_); } + + ssize_t n = handle_.read(&buffer_[0], chunk_size_); + if (n > 0) { + current_size_ = static_cast(n); + return true; + } + + current_size_ = 0; + finished_ = true; + return false; +} + +inline const char *Result::data() const { return buffer_.data(); } +inline size_t Result::size() const { return current_size_; } + +inline std::string Result::read_all() { + std::string result; + while (next()) { + result.append(data(), size()); + } + return result; +} + +} // namespace stream + +namespace sse { + +// SSEMessage implementations +inline SSEMessage::SSEMessage() : event("message") {} + +inline void SSEMessage::clear() { + event = "message"; + data.clear(); + id.clear(); +} + +// SSEClient implementations +inline SSEClient::SSEClient(Client &client, const std::string &path) + : client_(client), path_(path) {} + +inline SSEClient::SSEClient(Client &client, const std::string &path, + const Headers &headers) + : client_(client), path_(path), headers_(headers) {} + +inline SSEClient::~SSEClient() { stop(); } + +inline SSEClient &SSEClient::on_message(MessageHandler handler) { + on_message_ = std::move(handler); + return *this; +} + +inline SSEClient &SSEClient::on_event(const std::string &type, + MessageHandler handler) { + event_handlers_[type] = std::move(handler); + return *this; +} + +inline SSEClient &SSEClient::on_open(OpenHandler handler) { + on_open_ = std::move(handler); + return *this; +} + +inline SSEClient &SSEClient::on_error(ErrorHandler handler) { + on_error_ = std::move(handler); + return *this; +} + +inline SSEClient &SSEClient::set_reconnect_interval(int ms) { + reconnect_interval_ms_ = ms; + return *this; +} + +inline SSEClient &SSEClient::set_max_reconnect_attempts(int n) { + max_reconnect_attempts_ = n; + return *this; +} + +inline SSEClient &SSEClient::set_headers(const Headers &headers) { + std::lock_guard lock(headers_mutex_); + headers_ = headers; + return *this; +} + +inline bool SSEClient::is_connected() const { return connected_.load(); } + +inline const std::string &SSEClient::last_event_id() const { + return last_event_id_; +} + +inline void SSEClient::start() { + running_.store(true); + run_event_loop(); +} + +inline void SSEClient::start_async() { + running_.store(true); + async_thread_ = std::thread([this]() { run_event_loop(); }); +} + +inline void SSEClient::stop() { + running_.store(false); + client_.stop(); // Cancel any pending operations + if (async_thread_.joinable()) { async_thread_.join(); } +} + +inline bool SSEClient::parse_sse_line(const std::string &line, SSEMessage &msg, + int &retry_ms) { + // Blank line signals end of event + if (line.empty() || line == "\r") { return true; } + + // Lines starting with ':' are comments (ignored) + if (!line.empty() && line[0] == ':') { return false; } + + // Find the colon separator + auto colon_pos = line.find(':'); + if (colon_pos == std::string::npos) { + // Line with no colon is treated as field name with empty value + return false; + } + + auto field = line.substr(0, colon_pos); + std::string value; + + // Value starts after colon, skip optional single space + if (colon_pos + 1 < line.size()) { + auto value_start = colon_pos + 1; + if (line[value_start] == ' ') { value_start++; } + value = line.substr(value_start); + // Remove trailing \r if present + if (!value.empty() && value.back() == '\r') { value.pop_back(); } + } + + // Handle known fields + if (field == "event") { + msg.event = value; + } else if (field == "data") { + // Multiple data lines are concatenated with newlines + if (!msg.data.empty()) { msg.data += "\n"; } + msg.data += value; + } else if (field == "id") { + // Empty id is valid (clears the last event ID) + msg.id = value; + } else if (field == "retry") { + // Parse retry interval in milliseconds + { + int v = 0; + auto res = + detail::from_chars(value.data(), value.data() + value.size(), v); + if (res.ec == std::errc{}) { retry_ms = v; } + } + } + // Unknown fields are ignored per SSE spec + + return false; +} + +inline void SSEClient::run_event_loop() { + auto reconnect_count = 0; + + while (running_.load()) { + // Build headers, including Last-Event-ID if we have one + Headers request_headers; + { + std::lock_guard lock(headers_mutex_); + request_headers = headers_; + } + if (!last_event_id_.empty()) { + request_headers.emplace("Last-Event-ID", last_event_id_); + } + + // Open streaming connection + auto result = stream::Get(client_, path_, request_headers); + + // Connection error handling + if (!result) { + connected_.store(false); + if (on_error_) { on_error_(result.error()); } + + if (!should_reconnect(reconnect_count)) { break; } + wait_for_reconnect(); + reconnect_count++; + continue; + } + + if (result.status() != StatusCode::OK_200) { + connected_.store(false); + if (on_error_) { on_error_(Error::Connection); } + + // For certain errors, don't reconnect. + // Note: 401 is intentionally absent so that handlers can refresh + // credentials via set_headers() and let the client reconnect. + if (result.status() == StatusCode::NoContent_204 || + result.status() == StatusCode::NotFound_404 || + result.status() == StatusCode::Forbidden_403) { + break; + } + + if (!should_reconnect(reconnect_count)) { break; } + wait_for_reconnect(); + reconnect_count++; + continue; + } + + // Connection successful + connected_.store(true); + reconnect_count = 0; + if (on_open_) { on_open_(); } + + // Event receiving loop + std::string buffer; + SSEMessage current_msg; + + while (running_.load() && result.next()) { + buffer.append(result.data(), result.size()); + + // Process complete lines in the buffer + size_t line_start = 0; + size_t newline_pos; + + while ((newline_pos = buffer.find('\n', line_start)) != + std::string::npos) { + auto line = buffer.substr(line_start, newline_pos - line_start); + line_start = newline_pos + 1; + + // Parse the line and check if event is complete + auto event_complete = + parse_sse_line(line, current_msg, reconnect_interval_ms_); + + if (event_complete && !current_msg.data.empty()) { + // Update last_event_id for reconnection + if (!current_msg.id.empty()) { last_event_id_ = current_msg.id; } + + // Dispatch event to appropriate handler + dispatch_event(current_msg); + + current_msg.clear(); + } + } + + // Keep unprocessed data in buffer + buffer.erase(0, line_start); + } + + // Connection ended + connected_.store(false); + + if (!running_.load()) { break; } + + // Check for read errors + if (result.has_read_error()) { + if (on_error_) { on_error_(result.read_error()); } + } + + if (!should_reconnect(reconnect_count)) { break; } + wait_for_reconnect(); + reconnect_count++; + } + + connected_.store(false); +} + +inline void SSEClient::dispatch_event(const SSEMessage &msg) { + // Check for specific event type handler first + auto it = event_handlers_.find(msg.event); + if (it != event_handlers_.end()) { + it->second(msg); + return; + } + + // Fall back to generic message handler + if (on_message_) { on_message_(msg); } +} + +inline bool SSEClient::should_reconnect(int count) const { + if (!running_.load()) { return false; } + if (max_reconnect_attempts_ == 0) { return true; } // unlimited + return count < max_reconnect_attempts_; +} + +inline void SSEClient::wait_for_reconnect() { + // Use small increments to check running_ flag frequently + auto waited = 0; + while (running_.load() && waited < reconnect_interval_ms_) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + waited += 100; + } +} + +} // namespace sse + +#ifdef CPPHTTPLIB_SSL_ENABLED +/* + * TLS abstraction layer - internal function declarations + * These are implementation details and not part of the public API. + */ +namespace tls { + +// Client context +ctx_t create_client_context(); +void free_context(ctx_t ctx); +bool set_min_version(ctx_t ctx, Version version); +bool load_ca_pem(ctx_t ctx, const char *pem, size_t len); +bool load_ca_file(ctx_t ctx, const char *file_path); +bool load_ca_dir(ctx_t ctx, const char *dir_path); +bool load_system_certs(ctx_t ctx); +bool set_client_cert_pem(ctx_t ctx, const char *cert, const char *key, + const char *password); +bool set_client_cert_file(ctx_t ctx, const char *cert_path, + const char *key_path, const char *password); + +// Server context +ctx_t create_server_context(); +bool set_server_cert_pem(ctx_t ctx, const char *cert, const char *key, + const char *password); +bool set_server_cert_file(ctx_t ctx, const char *cert_path, + const char *key_path, const char *password); +bool set_client_ca_file(ctx_t ctx, const char *ca_file, const char *ca_dir); +void set_verify_client(ctx_t ctx, bool require); + +// Session management +session_t create_session(ctx_t ctx, socket_t sock); +void free_session(session_t session); +bool set_sni(session_t session, const char *hostname); +bool set_hostname(session_t session, const char *hostname); + +// Handshake (non-blocking capable) +TlsError connect(session_t session); +TlsError accept(session_t session); + +// Handshake with timeout (blocking until timeout) +bool connect_nonblocking(session_t session, socket_t sock, time_t timeout_sec, + time_t timeout_usec, TlsError *err); +bool accept_nonblocking(session_t session, socket_t sock, time_t timeout_sec, + time_t timeout_usec, TlsError *err); + +// I/O (non-blocking capable) +ssize_t read(session_t session, void *buf, size_t len, TlsError &err); +ssize_t write(session_t session, const void *buf, size_t len, TlsError &err); +int pending(const_session_t session); +void shutdown(session_t session, bool graceful); + +// Connection state +bool is_peer_closed(session_t session, socket_t sock); + +// Certificate verification +cert_t get_peer_cert(const_session_t session); +void free_cert(cert_t cert); +bool verify_hostname(cert_t cert, const char *hostname); +uint64_t hostname_mismatch_code(); +long get_verify_result(const_session_t session); + +// Certificate introspection +std::string get_cert_subject_cn(cert_t cert); +std::string get_cert_issuer_name(cert_t cert); +bool get_cert_sans(cert_t cert, std::vector &sans); +bool get_cert_validity(cert_t cert, time_t ¬_before, time_t ¬_after); +std::string get_cert_serial(cert_t cert); +bool get_cert_der(cert_t cert, std::vector &der); +const char *get_sni(const_session_t session); + +// CA store management +ca_store_t create_ca_store(const char *pem, size_t len); +void free_ca_store(ca_store_t store); +bool set_ca_store(ctx_t ctx, ca_store_t store); +size_t get_ca_certs(ctx_t ctx, std::vector &certs); +std::vector get_ca_names(ctx_t ctx); + +// Dynamic certificate update (for servers) +bool update_server_cert(ctx_t ctx, const char *cert_pem, const char *key_pem, + const char *password); +bool update_server_client_ca(ctx_t ctx, const char *ca_pem); + +// Certificate verification callback +bool set_verify_callback(ctx_t ctx, VerifyCallback callback); +long get_verify_error(const_session_t session); +std::string verify_error_string(long error_code); + +// TlsError information +uint64_t peek_error(); +uint64_t get_error(); +std::string error_string(uint64_t code); + +} // namespace tls +#endif // CPPHTTPLIB_SSL_ENABLED + +/* + * Group 1: detail namespace - Non-SSL utilities + */ + +namespace detail { + +inline bool set_socket_opt_impl(socket_t sock, int level, int optname, + const void *optval, socklen_t optlen) { + return setsockopt(sock, level, optname, +#ifdef _WIN32 + reinterpret_cast(optval), +#else + optval, +#endif + optlen) == 0; +} + +inline bool set_socket_opt_time(socket_t sock, int level, int optname, + time_t sec, time_t usec) { +#ifdef _WIN32 + auto timeout = static_cast(sec * 1000 + usec / 1000); +#else + timeval timeout; + timeout.tv_sec = static_cast(sec); + timeout.tv_usec = static_cast(usec); +#endif + return set_socket_opt_impl(sock, level, optname, &timeout, sizeof(timeout)); +} + +inline bool is_hex(char c, int &v) { + if (isdigit(static_cast(c))) { + v = c - '0'; + return true; + } else if ('A' <= c && c <= 'F') { + v = c - 'A' + 10; + return true; + } else if ('a' <= c && c <= 'f') { + v = c - 'a' + 10; + return true; + } + return false; +} + +inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt, + int &val) { + if (i >= s.size()) { return false; } + + val = 0; + for (; cnt; i++, cnt--) { + if (!s[i]) { return false; } + auto v = 0; + if (is_hex(s[i], v)) { + val = val * 16 + v; + } else { + return false; + } + } + return true; +} + +inline std::string from_i_to_hex(size_t n) { + static const auto charset = "0123456789abcdef"; + std::string ret; + do { + ret = charset[n & 15] + ret; + n >>= 4; + } while (n > 0); + return ret; +} + +inline std::string compute_etag(const FileStat &fs) { + if (!fs.is_file()) { return std::string(); } + + // If mtime cannot be determined (negative value indicates an error + // or sentinel), do not generate an ETag. Returning a neutral / fixed + // value like 0 could collide with a real file that legitimately has + // mtime == 0 (epoch) and lead to misleading validators. + auto mtime_raw = fs.mtime(); + if (mtime_raw < 0) { return std::string(); } + + auto mtime = static_cast(mtime_raw); + auto size = fs.size(); + + return std::string("W/\"") + from_i_to_hex(mtime) + "-" + + from_i_to_hex(size) + "\""; +} + +// Format time_t as HTTP-date (RFC 9110 Section 5.6.7): "Sun, 06 Nov 1994 +// 08:49:37 GMT" This implementation is defensive: it validates `mtime`, checks +// return values from `gmtime_r`/`gmtime_s`, and ensures `strftime` succeeds. +inline std::string file_mtime_to_http_date(time_t mtime) { + if (mtime < 0) { return std::string(); } + + struct tm tm_buf; +#ifdef _WIN32 + if (gmtime_s(&tm_buf, &mtime) != 0) { return std::string(); } +#else + if (gmtime_r(&mtime, &tm_buf) == nullptr) { return std::string(); } +#endif + char buf[64]; + if (strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", &tm_buf) == 0) { + return std::string(); + } + + return std::string(buf); +} + +// Parse HTTP-date (RFC 9110 Section 5.6.7) to time_t. Returns -1 on failure. +inline time_t parse_http_date(const std::string &date_str) { + struct tm tm_buf; + + // Create a classic locale object once for all parsing attempts + const std::locale classic_locale = std::locale::classic(); + + // Try to parse using std::get_time (C++11, cross-platform) + auto try_parse = [&](const char *fmt) -> bool { + std::istringstream ss(date_str); + ss.imbue(classic_locale); + + memset(&tm_buf, 0, sizeof(tm_buf)); + ss >> std::get_time(&tm_buf, fmt); + + return !ss.fail(); + }; + + // RFC 9110 preferred format (HTTP-date): "Sun, 06 Nov 1994 08:49:37 GMT" + if (!try_parse("%a, %d %b %Y %H:%M:%S")) { + // RFC 850 format: "Sunday, 06-Nov-94 08:49:37 GMT" + if (!try_parse("%A, %d-%b-%y %H:%M:%S")) { + // asctime format: "Sun Nov 6 08:49:37 1994" + if (!try_parse("%a %b %d %H:%M:%S %Y")) { + return static_cast(-1); + } + } + } + +#ifdef _WIN32 + return _mkgmtime(&tm_buf); +#elif defined _AIX + return mktime(&tm_buf); +#else + return timegm(&tm_buf); +#endif +} + +inline bool is_weak_etag(const std::string &s) { + // Check if the string is a weak ETag (starts with 'W/"') + return s.size() > 3 && s[0] == 'W' && s[1] == '/' && s[2] == '"'; +} + +inline bool is_strong_etag(const std::string &s) { + // Check if the string is a strong ETag (starts and ends with '"', at least 2 + // chars) + return s.size() >= 2 && s[0] == '"' && s.back() == '"'; +} + +inline size_t to_utf8(int code, char *buff) { + if (code < 0x0080) { + buff[0] = static_cast(code & 0x7F); + return 1; + } else if (code < 0x0800) { + buff[0] = static_cast(0xC0 | ((code >> 6) & 0x1F)); + buff[1] = static_cast(0x80 | (code & 0x3F)); + return 2; + } else if (code < 0xD800) { + buff[0] = static_cast(0xE0 | ((code >> 12) & 0xF)); + buff[1] = static_cast(0x80 | ((code >> 6) & 0x3F)); + buff[2] = static_cast(0x80 | (code & 0x3F)); + return 3; + } else if (code < 0xE000) { // D800 - DFFF is invalid... + return 0; + } else if (code < 0x10000) { + buff[0] = static_cast(0xE0 | ((code >> 12) & 0xF)); + buff[1] = static_cast(0x80 | ((code >> 6) & 0x3F)); + buff[2] = static_cast(0x80 | (code & 0x3F)); + return 3; + } else if (code < 0x110000) { + buff[0] = static_cast(0xF0 | ((code >> 18) & 0x7)); + buff[1] = static_cast(0x80 | ((code >> 12) & 0x3F)); + buff[2] = static_cast(0x80 | ((code >> 6) & 0x3F)); + buff[3] = static_cast(0x80 | (code & 0x3F)); + return 4; + } + + // NOTREACHED + return 0; +} + +} // namespace detail + +namespace ws { +namespace impl { + +inline bool is_valid_utf8(const std::string &s) { + size_t i = 0; + auto n = s.size(); + while (i < n) { + auto c = static_cast(s[i]); + size_t len; + uint32_t cp; + if (c < 0x80) { + i++; + continue; + } else if ((c & 0xE0) == 0xC0) { + len = 2; + cp = c & 0x1F; + } else if ((c & 0xF0) == 0xE0) { + len = 3; + cp = c & 0x0F; + } else if ((c & 0xF8) == 0xF0) { + len = 4; + cp = c & 0x07; + } else { + return false; + } + if (i + len > n) { return false; } + for (size_t j = 1; j < len; j++) { + auto b = static_cast(s[i + j]); + if ((b & 0xC0) != 0x80) { return false; } + cp = (cp << 6) | (b & 0x3F); + } + // Overlong encoding check + if (len == 2 && cp < 0x80) { return false; } + if (len == 3 && cp < 0x800) { return false; } + if (len == 4 && cp < 0x10000) { return false; } + // Surrogate halves (U+D800..U+DFFF) and beyond U+10FFFF are invalid + if (cp >= 0xD800 && cp <= 0xDFFF) { return false; } + if (cp > 0x10FFFF) { return false; } + i += len; + } + return true; +} + +} // namespace impl +} // namespace ws + +namespace detail { + +// NOTE: This code came up with the following stackoverflow post: +// https://stackoverflow.com/questions/180947/base64-decode-snippet-in-c +inline std::string base64_encode(const std::string &in) { + static const auto lookup = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + std::string out; + out.reserve(in.size()); + + auto val = 0; + auto valb = -6; + + for (auto c : in) { + val = (val << 8) + static_cast(c); + valb += 8; + while (valb >= 0) { + out.push_back(lookup[(val >> valb) & 0x3F]); + valb -= 6; + } + } + + if (valb > -6) { out.push_back(lookup[((val << 8) >> (valb + 8)) & 0x3F]); } + + while (out.size() % 4) { + out.push_back('='); + } + + return out; +} + +inline std::string sha1(const std::string &input) { + // RFC 3174 SHA-1 implementation + auto left_rotate = [](uint32_t x, uint32_t n) -> uint32_t { + return (x << n) | (x >> (32 - n)); + }; + + uint32_t h0 = 0x67452301; + uint32_t h1 = 0xEFCDAB89; + uint32_t h2 = 0x98BADCFE; + uint32_t h3 = 0x10325476; + uint32_t h4 = 0xC3D2E1F0; + + // Pre-processing: adding padding bits + std::string msg = input; + uint64_t original_bit_len = static_cast(msg.size()) * 8; + msg.push_back(static_cast(0x80u)); + while (msg.size() % 64 != 56) { + msg.push_back(0); + } + + // Append original length in bits as 64-bit big-endian + for (int i = 56; i >= 0; i -= 8) { + msg.push_back(static_cast((original_bit_len >> i) & 0xFF)); + } + + // Process each 512-bit chunk + for (size_t offset = 0; offset < msg.size(); offset += 64) { + uint32_t w[80]; + + for (size_t i = 0; i < 16; i++) { + w[i] = + (static_cast(static_cast(msg[offset + i * 4])) + << 24) | + (static_cast(static_cast(msg[offset + i * 4 + 1])) + << 16) | + (static_cast(static_cast(msg[offset + i * 4 + 2])) + << 8) | + (static_cast( + static_cast(msg[offset + i * 4 + 3]))); + } + + for (int i = 16; i < 80; i++) { + w[i] = left_rotate(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1); + } + + uint32_t a = h0, b = h1, c = h2, d = h3, e = h4; + + for (int i = 0; i < 80; i++) { + uint32_t f, k; + if (i < 20) { + f = (b & c) | ((~b) & d); + k = 0x5A827999; + } else if (i < 40) { + f = b ^ c ^ d; + k = 0x6ED9EBA1; + } else if (i < 60) { + f = (b & c) | (b & d) | (c & d); + k = 0x8F1BBCDC; + } else { + f = b ^ c ^ d; + k = 0xCA62C1D6; + } + + uint32_t temp = left_rotate(a, 5) + f + e + k + w[i]; + e = d; + d = c; + c = left_rotate(b, 30); + b = a; + a = temp; + } + + h0 += a; + h1 += b; + h2 += c; + h3 += d; + h4 += e; + } + + // Produce the final hash as a 20-byte binary string + std::string hash(20, '\0'); + for (size_t i = 0; i < 4; i++) { + hash[i] = static_cast((h0 >> (24 - i * 8)) & 0xFF); + hash[4 + i] = static_cast((h1 >> (24 - i * 8)) & 0xFF); + hash[8 + i] = static_cast((h2 >> (24 - i * 8)) & 0xFF); + hash[12 + i] = static_cast((h3 >> (24 - i * 8)) & 0xFF); + hash[16 + i] = static_cast((h4 >> (24 - i * 8)) & 0xFF); + } + return hash; +} + +inline std::string websocket_accept_key(const std::string &client_key) { + const std::string magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + return base64_encode(sha1(client_key + magic)); +} + +inline bool is_websocket_upgrade(const Request &req) { + if (req.method != "GET") { return false; } + + // Check Upgrade: websocket (case-insensitive) + auto upgrade_it = req.headers.find("Upgrade"); + if (upgrade_it == req.headers.end()) { return false; } + auto upgrade_val = case_ignore::to_lower(upgrade_it->second); + if (upgrade_val != "websocket") { return false; } + + // Check Connection header contains "Upgrade" + auto connection_it = req.headers.find("Connection"); + if (connection_it == req.headers.end()) { return false; } + auto connection_val = case_ignore::to_lower(connection_it->second); + if (connection_val.find("upgrade") == std::string::npos) { return false; } + + // Check Sec-WebSocket-Key is a valid base64-encoded 16-byte value (24 chars) + // RFC 6455 Section 4.2.1 + auto ws_key = req.get_header_value("Sec-WebSocket-Key"); + if (ws_key.size() != 24 || ws_key[22] != '=' || ws_key[23] != '=') { + return false; + } + static const std::string b64chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + for (size_t i = 0; i < 22; i++) { + if (b64chars.find(ws_key[i]) == std::string::npos) { return false; } + } + + // Check Sec-WebSocket-Version: 13 + auto version = req.get_header_value("Sec-WebSocket-Version"); + if (version != "13") { return false; } + + return true; +} + +inline bool write_websocket_frame(Stream &strm, ws::Opcode opcode, + const char *data, size_t len, bool fin, + bool mask) { + // First byte: FIN + opcode + uint8_t header[2]; + header[0] = static_cast((fin ? 0x80 : 0x00) | + (static_cast(opcode) & 0x0F)); + + // Second byte: MASK + payload length + if (len < 126) { + header[1] = static_cast(len); + if (mask) { header[1] |= 0x80; } + if (strm.write(reinterpret_cast(header), 2) < 0) { return false; } + } else if (len <= 0xFFFF) { + header[1] = 126; + if (mask) { header[1] |= 0x80; } + if (strm.write(reinterpret_cast(header), 2) < 0) { return false; } + uint8_t ext[2]; + ext[0] = static_cast((len >> 8) & 0xFF); + ext[1] = static_cast(len & 0xFF); + if (strm.write(reinterpret_cast(ext), 2) < 0) { return false; } + } else { + header[1] = 127; + if (mask) { header[1] |= 0x80; } + if (strm.write(reinterpret_cast(header), 2) < 0) { return false; } + uint8_t ext[8]; + for (int i = 7; i >= 0; i--) { + ext[7 - i] = + static_cast((static_cast(len) >> (i * 8)) & 0xFF); + } + if (strm.write(reinterpret_cast(ext), 8) < 0) { return false; } + } + + if (mask) { + // Generate random mask key + thread_local std::mt19937 rng(std::random_device{}()); + uint8_t mask_key[4]; + auto r = rng(); + std::memcpy(mask_key, &r, 4); + if (strm.write(reinterpret_cast(mask_key), 4) < 0) { return false; } + + // Write masked payload in chunks + const size_t chunk_size = 4096; + std::vector buf((std::min)(len, chunk_size)); + for (size_t offset = 0; offset < len; offset += chunk_size) { + size_t n = (std::min)(chunk_size, len - offset); + for (size_t i = 0; i < n; i++) { + buf[i] = + data[offset + i] ^ static_cast(mask_key[(offset + i) % 4]); + } + if (strm.write(buf.data(), n) < 0) { return false; } + } + } else { + if (len > 0) { + if (strm.write(data, len) < 0) { return false; } + } + } + + return true; +} + +} // namespace detail + +namespace ws { +namespace impl { + +inline bool read_websocket_frame(Stream &strm, Opcode &opcode, + std::string &payload, bool &fin, + bool expect_masked, size_t max_len) { + // Read first 2 bytes + uint8_t header[2]; + if (strm.read(reinterpret_cast(header), 2) != 2) { return false; } + + fin = (header[0] & 0x80) != 0; + + // RSV1, RSV2, RSV3 must be 0 when no extension is negotiated + if (header[0] & 0x70) { return false; } + + opcode = static_cast(header[0] & 0x0F); + bool masked = (header[1] & 0x80) != 0; + uint64_t payload_len = header[1] & 0x7F; + + // RFC 6455 Section 5.5: control frames MUST NOT be fragmented and + // MUST have a payload length of 125 bytes or less + bool is_control = (static_cast(opcode) & 0x08) != 0; + if (is_control) { + if (!fin) { return false; } + if (payload_len > 125) { return false; } + } + + if (masked != expect_masked) { return false; } + + // Extended payload length + if (payload_len == 126) { + uint8_t ext[2]; + if (strm.read(reinterpret_cast(ext), 2) != 2) { return false; } + payload_len = (static_cast(ext[0]) << 8) | ext[1]; + } else if (payload_len == 127) { + uint8_t ext[8]; + if (strm.read(reinterpret_cast(ext), 8) != 8) { return false; } + // RFC 6455 Section 5.2: the most significant bit MUST be 0 + if (ext[0] & 0x80) { return false; } + payload_len = 0; + for (int i = 0; i < 8; i++) { + payload_len = (payload_len << 8) | ext[i]; + } + } + + if (payload_len > max_len) { return false; } + + // Read mask key if present + uint8_t mask_key[4] = {0}; + if (masked) { + if (strm.read(reinterpret_cast(mask_key), 4) != 4) { return false; } + } + + // Read payload + payload.resize(static_cast(payload_len)); + if (payload_len > 0) { + size_t total_read = 0; + while (total_read < payload_len) { + auto n = strm.read(&payload[total_read], + static_cast(payload_len - total_read)); + if (n <= 0) { return false; } + total_read += static_cast(n); + } + } + + // Unmask if needed + if (masked) { + for (size_t i = 0; i < payload.size(); i++) { + payload[i] ^= static_cast(mask_key[i % 4]); + } + } + + return true; +} + +} // namespace impl +} // namespace ws + +namespace detail { + +inline bool is_valid_path(const std::string &path) { + size_t level = 0; + size_t i = 0; + + // Skip slash + while (i < path.size() && path[i] == '/') { + i++; + } + + while (i < path.size()) { + // Read component + auto beg = i; + while (i < path.size() && path[i] != '/') { + if (path[i] == '\0') { + return false; + } else if (path[i] == '\\') { + return false; + } + i++; + } + + auto len = i - beg; + assert(len > 0); + + if (!path.compare(beg, len, ".")) { + ; + } else if (!path.compare(beg, len, "..")) { + if (level == 0) { return false; } + level--; + } else { + level++; + } + + // Skip slash + while (i < path.size() && path[i] == '/') { + i++; + } + } + + return true; +} + +inline bool canonicalize_path(const char *path, std::string &resolved) { +#if defined(_WIN32) + char buf[_MAX_PATH]; + if (_fullpath(buf, path, _MAX_PATH) == nullptr) { return false; } + resolved = buf; +#elif defined(PATH_MAX) + char buf[PATH_MAX]; + if (realpath(path, buf) == nullptr) { return false; } + resolved = buf; +#else + auto buf = realpath(path, nullptr); + auto guard = scope_exit([&]() { std::free(buf); }); + if (buf == nullptr) { return false; } + resolved = buf; +#endif + return true; +} + +inline bool is_path_within_base(const std::string &resolved_path, + const std::string &resolved_base) { +#if defined(_WIN32) + return _strnicmp(resolved_path.c_str(), resolved_base.c_str(), + resolved_base.size()) == 0; +#else + return strncmp(resolved_path.c_str(), resolved_base.c_str(), + resolved_base.size()) == 0; +#endif +} + +inline FileStat::FileStat(const std::string &path) { +#if defined(_WIN32) + auto wpath = u8string_to_wstring(path.c_str()); + ret_ = _wstat(wpath.c_str(), &st_); +#else + ret_ = stat(path.c_str(), &st_); +#endif +} +inline bool FileStat::is_file() const { + return ret_ >= 0 && S_ISREG(st_.st_mode); +} +inline bool FileStat::is_dir() const { + return ret_ >= 0 && S_ISDIR(st_.st_mode); +} + +inline time_t FileStat::mtime() const { + return ret_ >= 0 ? static_cast(st_.st_mtime) + : static_cast(-1); +} + +inline size_t FileStat::size() const { + return ret_ >= 0 ? static_cast(st_.st_size) : 0; +} + +inline std::string encode_path(const std::string &s) { + std::string result; + result.reserve(s.size()); + + for (size_t i = 0; s[i]; i++) { + switch (s[i]) { + case ' ': result += "%20"; break; + case '+': result += "%2B"; break; + case '\r': result += "%0D"; break; + case '\n': result += "%0A"; break; + case '\'': result += "%27"; break; + case ',': result += "%2C"; break; + // case ':': result += "%3A"; break; // ok? probably... + case ';': result += "%3B"; break; + default: + auto c = static_cast(s[i]); + if (c >= 0x80) { + result += '%'; + char hex[4]; + auto len = snprintf(hex, sizeof(hex) - 1, "%02X", c); + assert(len == 2); + result.append(hex, static_cast(len)); + } else { + result += s[i]; + } + break; + } + } + + return result; +} + +inline std::string file_extension(const std::string &path) { + std::smatch m; + thread_local auto re = std::regex("\\.([a-zA-Z0-9]+)$"); + if (std::regex_search(path, m, re)) { return m[1].str(); } + return std::string(); +} + +inline bool is_space_or_tab(char c) { return c == ' ' || c == '\t'; } + +template +inline bool parse_header(const char *beg, const char *end, T fn); + +template +inline bool parse_header(const char *beg, const char *end, T fn) { + // Skip trailing spaces and tabs. + while (beg < end && is_space_or_tab(end[-1])) { + end--; + } + + auto p = beg; + while (p < end && *p != ':') { + p++; + } + + auto name = std::string(beg, p); + if (!detail::fields::is_field_name(name)) { return false; } + + if (p == end) { return false; } + + auto key_end = p; + + if (*p++ != ':') { return false; } + + while (p < end && is_space_or_tab(*p)) { + p++; + } + + if (p <= end) { + auto key_len = key_end - beg; + if (!key_len) { return false; } + + auto key = std::string(beg, key_end); + auto val = std::string(p, end); + + if (!detail::fields::is_field_value(val)) { return false; } + + // RFC 9110 §5.5: header field values are opaque octets and MUST NOT be + // percent-decoded by the recipient. Applications that need to interpret a + // value as a URI component should call httplib::decode_uri_component() + // (or decode_path_component()) explicitly. + fn(key, val); + + return true; + } + + return false; +} + +inline bool parse_trailers(stream_line_reader &line_reader, Headers &dest, + const Headers &src_headers) { + // NOTE: In RFC 9112, '7.1 Chunked Transfer Coding' mentions "The chunked + // transfer coding is complete when a chunk with a chunk-size of zero is + // received, possibly followed by a trailer section, and finally terminated by + // an empty line". https://www.rfc-editor.org/rfc/rfc9112.html#section-7.1 + // + // In '7.1.3. Decoding Chunked', however, the pseudo-code in the section + // doesn't care for the existence of the final CRLF. In other words, it seems + // to be ok whether the final CRLF exists or not in the chunked data. + // https://www.rfc-editor.org/rfc/rfc9112.html#section-7.1.3 + // + // According to the reference code in RFC 9112, cpp-httplib now allows + // chunked transfer coding data without the final CRLF. + + // RFC 7230 Section 4.1.2 - Headers prohibited in trailers + thread_local case_ignore::unordered_set prohibited_trailers = { + "transfer-encoding", + "content-length", + "host", + "authorization", + "www-authenticate", + "proxy-authenticate", + "proxy-authorization", + "cookie", + "set-cookie", + "cache-control", + "expect", + "max-forwards", + "pragma", + "range", + "te", + "age", + "expires", + "date", + "location", + "retry-after", + "vary", + "warning", + "content-encoding", + "content-type", + "content-range", + "trailer"}; + + case_ignore::unordered_set declared_trailers; + auto trailer_header = get_header_value(src_headers, "Trailer", "", 0); + if (trailer_header && std::strlen(trailer_header)) { + auto len = std::strlen(trailer_header); + split(trailer_header, trailer_header + len, ',', + [&](const char *b, const char *e) { + const char *kbeg = b; + const char *kend = e; + while (kbeg < kend && (*kbeg == ' ' || *kbeg == '\t')) { + ++kbeg; + } + while (kend > kbeg && (kend[-1] == ' ' || kend[-1] == '\t')) { + --kend; + } + std::string key(kbeg, static_cast(kend - kbeg)); + if (!key.empty() && + prohibited_trailers.find(key) == prohibited_trailers.end()) { + declared_trailers.insert(key); + } + }); + } + + size_t trailer_header_count = 0; + while (strcmp(line_reader.ptr(), "\r\n") != 0) { + if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; } + if (trailer_header_count >= CPPHTTPLIB_HEADER_MAX_COUNT) { return false; } + + constexpr auto line_terminator_len = 2; + auto line_beg = line_reader.ptr(); + auto line_end = + line_reader.ptr() + line_reader.size() - line_terminator_len; + + if (!parse_header(line_beg, line_end, + [&](const std::string &key, const std::string &val) { + if (declared_trailers.find(key) != + declared_trailers.end()) { + dest.emplace(key, val); + trailer_header_count++; + } + })) { + return false; + } + + if (!line_reader.getline()) { return false; } + } + + return true; +} + +inline std::pair trim(const char *b, const char *e, size_t left, + size_t right) { + while (b + left < e && is_space_or_tab(b[left])) { + left++; + } + while (right > 0 && is_space_or_tab(b[right - 1])) { + right--; + } + return std::make_pair(left, right); +} + +inline std::string trim_copy(const std::string &s) { + auto r = trim(s.data(), s.data() + s.size(), 0, s.size()); + return s.substr(r.first, r.second - r.first); +} + +inline std::string trim_double_quotes_copy(const std::string &s) { + if (s.length() >= 2 && s.front() == '"' && s.back() == '"') { + return s.substr(1, s.size() - 2); + } + return s; +} + +inline void +divide(const char *data, std::size_t size, char d, + std::function + fn) { + const auto it = std::find(data, data + size, d); + const auto found = static_cast(it != data + size); + const auto lhs_data = data; + const auto lhs_size = static_cast(it - data); + const auto rhs_data = it + found; + const auto rhs_size = size - lhs_size - found; + + fn(lhs_data, lhs_size, rhs_data, rhs_size); +} + +inline void +divide(const std::string &str, char d, + std::function + fn) { + divide(str.data(), str.size(), d, std::move(fn)); +} + +inline void split(const char *b, const char *e, char d, + std::function fn) { + return split(b, e, d, (std::numeric_limits::max)(), std::move(fn)); +} + +inline void split(const char *b, const char *e, char d, size_t m, + std::function fn) { + size_t i = 0; + size_t beg = 0; + size_t count = 1; + + while (e ? (b + i < e) : (b[i] != '\0')) { + if (b[i] == d && count < m) { + auto r = trim(b, e, beg, i); + if (r.first < r.second) { fn(&b[r.first], &b[r.second]); } + beg = i + 1; + count++; + } + i++; + } + + if (i) { + auto r = trim(b, e, beg, i); + if (r.first < r.second) { fn(&b[r.first], &b[r.second]); } + } +} + +inline bool split_find(const char *b, const char *e, char d, size_t m, + std::function fn) { + size_t i = 0; + size_t beg = 0; + size_t count = 1; + + while (e ? (b + i < e) : (b[i] != '\0')) { + if (b[i] == d && count < m) { + auto r = trim(b, e, beg, i); + if (r.first < r.second) { + auto found = fn(&b[r.first], &b[r.second]); + if (found) { return true; } + } + beg = i + 1; + count++; + } + i++; + } + + if (i) { + auto r = trim(b, e, beg, i); + if (r.first < r.second) { + auto found = fn(&b[r.first], &b[r.second]); + if (found) { return true; } + } + } + + return false; +} + +inline bool split_find(const char *b, const char *e, char d, + std::function fn) { + return split_find(b, e, d, (std::numeric_limits::max)(), + std::move(fn)); +} + +inline stream_line_reader::stream_line_reader(Stream &strm, char *fixed_buffer, + size_t fixed_buffer_size) + : strm_(strm), fixed_buffer_(fixed_buffer), + fixed_buffer_size_(fixed_buffer_size) {} + +inline const char *stream_line_reader::ptr() const { + if (growable_buffer_.empty()) { + return fixed_buffer_; + } else { + return growable_buffer_.data(); + } +} + +inline size_t stream_line_reader::size() const { + if (growable_buffer_.empty()) { + return fixed_buffer_used_size_; + } else { + return growable_buffer_.size(); + } +} + +inline bool stream_line_reader::end_with_crlf() const { + auto end = ptr() + size(); + return size() >= 2 && end[-2] == '\r' && end[-1] == '\n'; +} + +inline bool stream_line_reader::getline() { + fixed_buffer_used_size_ = 0; + growable_buffer_.clear(); + +#ifndef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR + char prev_byte = 0; +#endif + + for (size_t i = 0;; i++) { + if (size() >= CPPHTTPLIB_MAX_LINE_LENGTH) { + // Treat exceptionally long lines as an error to + // prevent infinite loops/memory exhaustion + return false; + } + char byte; + auto n = strm_.read(&byte, 1); + + if (n < 0) { + return false; + } else if (n == 0) { + if (i == 0) { + return false; + } else { + break; + } + } + + append(byte); + +#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR + if (byte == '\n') { break; } +#else + if (prev_byte == '\r' && byte == '\n') { break; } + prev_byte = byte; +#endif + } + + return true; +} + +inline void stream_line_reader::append(char c) { + if (fixed_buffer_used_size_ < fixed_buffer_size_ - 1) { + fixed_buffer_[fixed_buffer_used_size_++] = c; + fixed_buffer_[fixed_buffer_used_size_] = '\0'; + } else { + if (growable_buffer_.empty()) { + assert(fixed_buffer_[fixed_buffer_used_size_] == '\0'); + growable_buffer_.assign(fixed_buffer_, fixed_buffer_used_size_); + } + growable_buffer_ += c; + } +} + +inline mmap::mmap(const char *path) { open(path); } + +inline mmap::~mmap() { close(); } + +inline bool mmap::open(const char *path) { + close(); + +#if defined(_WIN32) + auto wpath = u8string_to_wstring(path); + if (wpath.empty()) { return false; } + + hFile_ = ::CreateFileW(wpath.c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + + if (hFile_ == INVALID_HANDLE_VALUE) { return false; } + + LARGE_INTEGER size{}; + if (!::GetFileSizeEx(hFile_, &size)) { return false; } + // If the following line doesn't compile due to QuadPart, update Windows SDK. + // See: + // https://github.com/yhirose/cpp-httplib/issues/1903#issuecomment-2316520721 + if (static_cast(size.QuadPart) > + (std::numeric_limits::max)()) { + // `size_t` might be 32-bits, on 32-bits Windows. + return false; + } + size_ = static_cast(size.QuadPart); + + hMapping_ = ::CreateFileMappingW(hFile_, nullptr, PAGE_READONLY, + size.HighPart, size.LowPart, nullptr); + + // Special treatment for an empty file... + if (hMapping_ == NULL && size_ == 0) { + close(); + is_open_empty_file = true; + return true; + } + + if (hMapping_ == NULL) { + close(); + return false; + } + + addr_ = ::MapViewOfFile(hMapping_, FILE_MAP_READ, 0, 0, 0); + + if (addr_ == nullptr) { + close(); + return false; + } +#else + fd_ = ::open(path, O_RDONLY); + if (fd_ == -1) { return false; } + + struct stat sb; + if (fstat(fd_, &sb) == -1) { + close(); + return false; + } + size_ = static_cast(sb.st_size); + + addr_ = ::mmap(NULL, size_, PROT_READ, MAP_PRIVATE, fd_, 0); + + // Special treatment for an empty file... + if (addr_ == MAP_FAILED && size_ == 0) { + close(); + is_open_empty_file = true; + return false; + } +#endif + + return true; +} + +inline bool mmap::is_open() const { + return is_open_empty_file ? true : addr_ != nullptr; +} + +inline size_t mmap::size() const { return size_; } + +inline const char *mmap::data() const { + return is_open_empty_file ? "" : static_cast(addr_); +} + +inline void mmap::close() { +#if defined(_WIN32) + if (addr_) { + ::UnmapViewOfFile(addr_); + addr_ = nullptr; + } + + if (hMapping_) { + ::CloseHandle(hMapping_); + hMapping_ = NULL; + } + + if (hFile_ != INVALID_HANDLE_VALUE) { + ::CloseHandle(hFile_); + hFile_ = INVALID_HANDLE_VALUE; + } + + is_open_empty_file = false; +#else + if (addr_ != nullptr) { + munmap(addr_, size_); + addr_ = nullptr; + } + + if (fd_ != -1) { + ::close(fd_); + fd_ = -1; + } +#endif + size_ = 0; +} +inline int close_socket(socket_t sock) noexcept { +#ifdef _WIN32 + return closesocket(sock); +#else + return close(sock); +#endif +} + +template inline ssize_t handle_EINTR(T fn) { + ssize_t res = 0; + while (true) { + res = fn(); + if (res < 0 && errno == EINTR) { + std::this_thread::sleep_for(std::chrono::microseconds{1}); + continue; + } + break; + } + return res; +} + +inline ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags) { + return handle_EINTR([&]() { + return recv(sock, +#ifdef _WIN32 + static_cast(ptr), static_cast(size), +#else + ptr, size, +#endif + flags); + }); +} + +inline ssize_t send_socket(socket_t sock, const void *ptr, size_t size, + int flags) { + return handle_EINTR([&]() { + return send(sock, +#ifdef _WIN32 + static_cast(ptr), static_cast(size), +#else + ptr, size, +#endif + flags); + }); +} + +inline int poll_wrapper(struct pollfd *fds, nfds_t nfds, int timeout) { +#ifdef _WIN32 + return ::WSAPoll(fds, nfds, timeout); +#else + return ::poll(fds, nfds, timeout); +#endif +} + +inline ssize_t select_impl(socket_t sock, short events, time_t sec, + time_t usec) { + struct pollfd pfd; + pfd.fd = sock; + pfd.events = events; + pfd.revents = 0; + + auto timeout = static_cast(sec * 1000 + usec / 1000); + + return handle_EINTR([&]() { return poll_wrapper(&pfd, 1, timeout); }); +} + +inline ssize_t select_read(socket_t sock, time_t sec, time_t usec) { + return select_impl(sock, POLLIN, sec, usec); +} + +inline ssize_t select_write(socket_t sock, time_t sec, time_t usec) { + return select_impl(sock, POLLOUT, sec, usec); +} + +inline Error wait_until_socket_is_ready(socket_t sock, time_t sec, + time_t usec) { + struct pollfd pfd_read; + pfd_read.fd = sock; + pfd_read.events = POLLIN | POLLOUT; + pfd_read.revents = 0; + + auto timeout = static_cast(sec * 1000 + usec / 1000); + + auto poll_res = + handle_EINTR([&]() { return poll_wrapper(&pfd_read, 1, timeout); }); + + if (poll_res == 0) { return Error::ConnectionTimeout; } + + if (poll_res > 0 && pfd_read.revents & (POLLIN | POLLOUT)) { + auto error = 0; + socklen_t len = sizeof(error); + auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR, + reinterpret_cast(&error), &len); + auto successful = res >= 0 && !error; + return successful ? Error::Success : Error::Connection; + } + + return Error::Connection; +} + +inline bool is_socket_alive(socket_t sock) { + const auto val = detail::select_read(sock, 0, 0); + if (val == 0) { + return true; + } else if (val < 0 && errno == EBADF) { + return false; + } + char buf[1]; + return detail::read_socket(sock, &buf[0], sizeof(buf), MSG_PEEK) > 0; +} + +class SocketStream final : public Stream { +public: + SocketStream(socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, time_t write_timeout_usec, + time_t max_timeout_msec = 0, + std::chrono::time_point start_time = + (std::chrono::steady_clock::time_point::min)()); + ~SocketStream() override; + + bool is_readable() const override; + bool wait_readable() const override; + bool wait_writable() const override; + bool is_peer_alive() const override; + ssize_t read(char *ptr, size_t size) override; + ssize_t write(const char *ptr, size_t size) override; + void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; + socket_t socket() const override; + time_t duration() const override; + void set_read_timeout(time_t sec, time_t usec = 0) override; + +private: + socket_t sock_; + time_t read_timeout_sec_; + time_t read_timeout_usec_; + time_t write_timeout_sec_; + time_t write_timeout_usec_; + time_t max_timeout_msec_; + const std::chrono::time_point start_time_; + + std::vector read_buff_; + size_t read_buff_off_ = 0; + size_t read_buff_content_size_ = 0; + + static const size_t read_buff_size_ = 1024l * 4; +}; + +inline bool keep_alive(const std::atomic &svr_sock, socket_t sock, + time_t keep_alive_timeout_sec) { + using namespace std::chrono; + + const auto interval_usec = + CPPHTTPLIB_KEEPALIVE_TIMEOUT_CHECK_INTERVAL_USECOND; + + // Avoid expensive `steady_clock::now()` call for the first time + if (select_read(sock, 0, interval_usec) > 0) { return true; } + + const auto start = steady_clock::now() - microseconds{interval_usec}; + const auto timeout = seconds{keep_alive_timeout_sec}; + + while (true) { + if (svr_sock == INVALID_SOCKET) { + break; // Server socket is closed + } + + auto val = select_read(sock, 0, interval_usec); + if (val < 0) { + break; // Ssocket error + } else if (val == 0) { + if (steady_clock::now() - start > timeout) { + break; // Timeout + } + } else { + return true; // Ready for read + } + } + + return false; +} + +template +inline bool +process_server_socket_core(const std::atomic &svr_sock, socket_t sock, + size_t keep_alive_max_count, + time_t keep_alive_timeout_sec, T callback) { + assert(keep_alive_max_count > 0); + auto ret = false; + auto count = keep_alive_max_count; + while (count > 0 && keep_alive(svr_sock, sock, keep_alive_timeout_sec)) { + auto close_connection = count == 1; + auto connection_closed = false; + ret = callback(close_connection, connection_closed); + if (!ret || connection_closed) { break; } + count--; + } + return ret; +} + +template +inline bool +process_server_socket(const std::atomic &svr_sock, socket_t sock, + size_t keep_alive_max_count, + time_t keep_alive_timeout_sec, time_t read_timeout_sec, + time_t read_timeout_usec, time_t write_timeout_sec, + time_t write_timeout_usec, T callback) { + return process_server_socket_core( + svr_sock, sock, keep_alive_max_count, keep_alive_timeout_sec, + [&](bool close_connection, bool &connection_closed) { + SocketStream strm(sock, read_timeout_sec, read_timeout_usec, + write_timeout_sec, write_timeout_usec); + return callback(strm, close_connection, connection_closed); + }); +} + +inline bool process_client_socket( + socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, time_t write_timeout_usec, + time_t max_timeout_msec, + std::chrono::time_point start_time, + std::function callback) { + SocketStream strm(sock, read_timeout_sec, read_timeout_usec, + write_timeout_sec, write_timeout_usec, max_timeout_msec, + start_time); + return callback(strm); +} + +inline int shutdown_socket(socket_t sock) noexcept { +#ifdef _WIN32 + return shutdown(sock, SD_BOTH); +#else + return shutdown(sock, SHUT_RDWR); +#endif +} + +inline std::string escape_abstract_namespace_unix_domain(const std::string &s) { + if (s.size() > 1 && s[0] == '\0') { + auto ret = s; + ret[0] = '@'; + return ret; + } + return s; +} + +inline std::string +unescape_abstract_namespace_unix_domain(const std::string &s) { + if (s.size() > 1 && s[0] == '@') { + auto ret = s; + ret[0] = '\0'; + return ret; + } + return s; +} + +inline int getaddrinfo_with_timeout(const char *node, const char *service, + const struct addrinfo *hints, + struct addrinfo **res, time_t timeout_sec) { +#ifdef CPPHTTPLIB_USE_NON_BLOCKING_GETADDRINFO + if (timeout_sec <= 0) { + // No timeout specified, use standard getaddrinfo + return getaddrinfo(node, service, hints, res); + } + +#ifdef _WIN32 + // Windows-specific implementation using GetAddrInfoEx with overlapped I/O + OVERLAPPED overlapped = {}; + HANDLE event = CreateEventW(nullptr, TRUE, FALSE, nullptr); + if (!event) { return EAI_FAIL; } + + overlapped.hEvent = event; + + PADDRINFOEXW result_addrinfo = nullptr; + HANDLE cancel_handle = nullptr; + + ADDRINFOEXW hints_ex = {}; + if (hints) { + hints_ex.ai_flags = hints->ai_flags; + hints_ex.ai_family = hints->ai_family; + hints_ex.ai_socktype = hints->ai_socktype; + hints_ex.ai_protocol = hints->ai_protocol; + } + + auto wnode = u8string_to_wstring(node); + auto wservice = u8string_to_wstring(service); + + auto ret = ::GetAddrInfoExW(wnode.data(), wservice.data(), NS_DNS, nullptr, + hints ? &hints_ex : nullptr, &result_addrinfo, + nullptr, &overlapped, nullptr, &cancel_handle); + + if (ret == WSA_IO_PENDING) { + auto wait_result = + ::WaitForSingleObject(event, static_cast(timeout_sec * 1000)); + if (wait_result == WAIT_TIMEOUT) { + if (cancel_handle) { ::GetAddrInfoExCancel(&cancel_handle); } + ::CloseHandle(event); + return EAI_AGAIN; + } + + DWORD bytes_returned; + if (!::GetOverlappedResult((HANDLE)INVALID_SOCKET, &overlapped, + &bytes_returned, FALSE)) { + ::CloseHandle(event); + return ::WSAGetLastError(); + } + } + + ::CloseHandle(event); + + if (ret == NO_ERROR || ret == WSA_IO_PENDING) { + *res = reinterpret_cast(result_addrinfo); + return 0; + } + + return ret; +#elif TARGET_OS_MAC && defined(__clang__) + if (!node) { return EAI_NONAME; } + // macOS implementation using CFHost API for asynchronous DNS resolution + CFStringRef hostname_ref = CFStringCreateWithCString( + kCFAllocatorDefault, node, kCFStringEncodingUTF8); + if (!hostname_ref) { return EAI_MEMORY; } + + CFHostRef host_ref = CFHostCreateWithName(kCFAllocatorDefault, hostname_ref); + CFRelease(hostname_ref); + if (!host_ref) { return EAI_MEMORY; } + + // Set up context for callback + struct CFHostContext { + bool completed = false; + bool success = false; + CFArrayRef addresses = nullptr; + std::mutex mutex; + std::condition_variable cv; + } context; + + CFHostClientContext client_context; + memset(&client_context, 0, sizeof(client_context)); + client_context.info = &context; + + // Set callback + auto callback = [](CFHostRef theHost, CFHostInfoType /*typeInfo*/, + const CFStreamError *error, void *info) { + auto ctx = static_cast(info); + std::lock_guard lock(ctx->mutex); + + if (error && error->error != 0) { + ctx->success = false; + } else { + Boolean hasBeenResolved; + ctx->addresses = CFHostGetAddressing(theHost, &hasBeenResolved); + if (ctx->addresses && hasBeenResolved) { + CFRetain(ctx->addresses); + ctx->success = true; + } else { + ctx->success = false; + } + } + ctx->completed = true; + ctx->cv.notify_one(); + }; + + if (!CFHostSetClient(host_ref, callback, &client_context)) { + CFRelease(host_ref); + return EAI_SYSTEM; + } + + // Schedule on run loop + CFRunLoopRef run_loop = CFRunLoopGetCurrent(); + CFHostScheduleWithRunLoop(host_ref, run_loop, kCFRunLoopDefaultMode); + + // Start resolution + CFStreamError stream_error; + if (!CFHostStartInfoResolution(host_ref, kCFHostAddresses, &stream_error)) { + CFHostUnscheduleFromRunLoop(host_ref, run_loop, kCFRunLoopDefaultMode); + CFRelease(host_ref); + return EAI_FAIL; + } + + // Wait for completion with timeout + auto timeout_time = + std::chrono::steady_clock::now() + std::chrono::seconds(timeout_sec); + bool timed_out = false; + + { + std::unique_lock lock(context.mutex); + + while (!context.completed) { + auto now = std::chrono::steady_clock::now(); + if (now >= timeout_time) { + timed_out = true; + break; + } + + // Run the runloop for a short time + lock.unlock(); + CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, true); + lock.lock(); + } + } + + // Clean up + CFHostUnscheduleFromRunLoop(host_ref, run_loop, kCFRunLoopDefaultMode); + CFHostSetClient(host_ref, nullptr, nullptr); + + if (timed_out || !context.completed) { + CFHostCancelInfoResolution(host_ref, kCFHostAddresses); + CFRelease(host_ref); + return EAI_AGAIN; + } + + if (!context.success || !context.addresses) { + CFRelease(host_ref); + return EAI_NODATA; + } + + // Convert CFArray to addrinfo + CFIndex count = CFArrayGetCount(context.addresses); + if (count == 0) { + CFRelease(context.addresses); + CFRelease(host_ref); + return EAI_NODATA; + } + + struct addrinfo *result_addrinfo = nullptr; + struct addrinfo **current = &result_addrinfo; + + for (CFIndex i = 0; i < count; i++) { + CFDataRef addr_data = + static_cast(CFArrayGetValueAtIndex(context.addresses, i)); + if (!addr_data) continue; + + const struct sockaddr *sockaddr_ptr = + reinterpret_cast(CFDataGetBytePtr(addr_data)); + socklen_t sockaddr_len = static_cast(CFDataGetLength(addr_data)); + + // Allocate addrinfo structure + *current = static_cast(malloc(sizeof(struct addrinfo))); + if (!*current) { + freeaddrinfo(result_addrinfo); + CFRelease(context.addresses); + CFRelease(host_ref); + return EAI_MEMORY; + } + + memset(*current, 0, sizeof(struct addrinfo)); + + // Set up addrinfo fields + (*current)->ai_family = sockaddr_ptr->sa_family; + (*current)->ai_socktype = hints ? hints->ai_socktype : SOCK_STREAM; + (*current)->ai_protocol = hints ? hints->ai_protocol : IPPROTO_TCP; + (*current)->ai_addrlen = sockaddr_len; + + // Copy sockaddr + (*current)->ai_addr = static_cast(malloc(sockaddr_len)); + if (!(*current)->ai_addr) { + freeaddrinfo(result_addrinfo); + CFRelease(context.addresses); + CFRelease(host_ref); + return EAI_MEMORY; + } + memcpy((*current)->ai_addr, sockaddr_ptr, sockaddr_len); + + // Set port if service is specified + if (service && *service) { + int port = 0; + if (parse_port(service, strlen(service), port)) { + if (sockaddr_ptr->sa_family == AF_INET) { + reinterpret_cast((*current)->ai_addr) + ->sin_port = htons(static_cast(port)); + } else if (sockaddr_ptr->sa_family == AF_INET6) { + reinterpret_cast((*current)->ai_addr) + ->sin6_port = htons(static_cast(port)); + } + } + } + + current = &((*current)->ai_next); + } + + CFRelease(context.addresses); + CFRelease(host_ref); + + *res = result_addrinfo; + return 0; +#elif defined(_GNU_SOURCE) && defined(__GLIBC__) && \ + (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 2)) + // #2431: gai_cancel() is non-blocking and may return EAI_NOTCANCELED while + // the resolver worker still references the stack-local gaicb. The cancel + // path therefore waits (gai_suspend with no timeout) for the worker to + // actually finish before letting the stack frame go. The trade-off is that + // a wedged DNS server can hold this thread for the system resolver timeout + // (~30s by default) past the caller's connection timeout. + struct gaicb request {}; + struct gaicb *requests[1] = {&request}; + struct sigevent sevp {}; + struct timespec timeout { + timeout_sec, 0 + }; + + request.ar_name = node; + request.ar_service = service; + request.ar_request = hints; + sevp.sigev_notify = SIGEV_NONE; + + int rc = getaddrinfo_a(GAI_NOWAIT, requests, 1, &sevp); + if (rc != 0) { return rc; } + + auto cleanup = scope_exit([&] { + if (request.ar_result) { freeaddrinfo(request.ar_result); } + }); + + int wait_result = gai_suspend(requests, 1, &timeout); + + if (wait_result == 0 || wait_result == EAI_ALLDONE) { + int gai_result = gai_error(&request); + if (gai_result == 0) { + *res = request.ar_result; + request.ar_result = nullptr; + return 0; + } + return gai_result; + } + + gai_cancel(&request); + while (gai_error(&request) == EAI_INPROGRESS) { + gai_suspend(requests, 1, nullptr); + } + return wait_result; +#else + // Fallback implementation using thread-based timeout for other Unix systems. + + struct GetAddrInfoState { + ~GetAddrInfoState() { + if (info) { freeaddrinfo(info); } + } + + std::mutex mutex; + std::condition_variable result_cv; + bool completed = false; + int result = EAI_SYSTEM; + std::string node; + std::string service; + struct addrinfo hints; + struct addrinfo *info = nullptr; + }; + + // Allocate on the heap, so the resolver thread can keep using the data. + auto state = std::make_shared(); + if (node) { state->node = node; } + state->service = service; + state->hints = *hints; + + std::thread resolve_thread([state]() { + auto thread_result = + getaddrinfo(state->node.c_str(), state->service.c_str(), &state->hints, + &state->info); + + std::lock_guard lock(state->mutex); + state->result = thread_result; + state->completed = true; + state->result_cv.notify_one(); + }); + + // Wait for completion or timeout + std::unique_lock lock(state->mutex); + auto finished = + state->result_cv.wait_for(lock, std::chrono::seconds(timeout_sec), + [&] { return state->completed; }); + + if (finished) { + // Operation completed within timeout + resolve_thread.join(); + *res = state->info; + state->info = nullptr; // Pass ownership to caller + return state->result; + } else { + // Timeout occurred + resolve_thread.detach(); // Let the thread finish in background + return EAI_AGAIN; // Return timeout error + } +#endif +#else + (void)(timeout_sec); // Unused parameter for non-blocking getaddrinfo + return getaddrinfo(node, service, hints, res); +#endif +} + +template +socket_t create_socket(const std::string &host, const std::string &ip, int port, + int address_family, int socket_flags, bool tcp_nodelay, + bool ipv6_v6only, SocketOptions socket_options, + BindOrConnect bind_or_connect, time_t timeout_sec = 0) { + // Get address info + const char *node = nullptr; + struct addrinfo hints; + struct addrinfo *result; + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = IPPROTO_IP; + + if (!ip.empty()) { + node = ip.c_str(); + // Ask getaddrinfo to convert IP in c-string to address + hints.ai_family = AF_UNSPEC; + hints.ai_flags = AI_NUMERICHOST; + } else { + if (!host.empty()) { node = host.c_str(); } + hints.ai_family = address_family; + hints.ai_flags = socket_flags; + } + +#if !defined(_WIN32) || defined(CPPHTTPLIB_HAVE_AFUNIX_H) + if (hints.ai_family == AF_UNIX) { + const auto addrlen = host.length(); + if (addrlen > sizeof(sockaddr_un::sun_path)) { return INVALID_SOCKET; } + +#ifdef SOCK_CLOEXEC + auto sock = socket(hints.ai_family, hints.ai_socktype | SOCK_CLOEXEC, + hints.ai_protocol); +#else + auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol); +#endif + + if (sock != INVALID_SOCKET) { + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + + auto unescaped_host = unescape_abstract_namespace_unix_domain(host); + std::copy(unescaped_host.begin(), unescaped_host.end(), addr.sun_path); + + hints.ai_addr = reinterpret_cast(&addr); + hints.ai_addrlen = static_cast( + sizeof(addr) - sizeof(addr.sun_path) + addrlen); + +#ifndef SOCK_CLOEXEC +#ifndef _WIN32 + fcntl(sock, F_SETFD, FD_CLOEXEC); +#endif +#endif + + if (socket_options) { socket_options(sock); } + +#ifdef _WIN32 + // Setting SO_REUSEADDR seems not to work well with AF_UNIX on windows, so + // remove the option. + set_socket_opt(sock, SOL_SOCKET, SO_REUSEADDR, 0); +#endif + + bool dummy; + if (!bind_or_connect(sock, hints, dummy)) { + close_socket(sock); + sock = INVALID_SOCKET; + } + } + return sock; + } +#endif + + auto service = std::to_string(port); + + if (getaddrinfo_with_timeout(node, service.c_str(), &hints, &result, + timeout_sec)) { +#if defined __linux__ && !defined __ANDROID__ + res_init(); +#endif + return INVALID_SOCKET; + } + auto se = detail::scope_exit([&] { freeaddrinfo(result); }); + + for (auto rp = result; rp; rp = rp->ai_next) { + // Create a socket +#ifdef _WIN32 + auto sock = + WSASocketW(rp->ai_family, rp->ai_socktype, rp->ai_protocol, nullptr, 0, + WSA_FLAG_NO_HANDLE_INHERIT | WSA_FLAG_OVERLAPPED); + /** + * Since the WSA_FLAG_NO_HANDLE_INHERIT is only supported on Windows 7 SP1 + * and above the socket creation fails on older Windows Systems. + * + * Let's try to create a socket the old way in this case. + * + * Reference: + * https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-wsasocketa + * + * WSA_FLAG_NO_HANDLE_INHERIT: + * This flag is supported on Windows 7 with SP1, Windows Server 2008 R2 with + * SP1, and later + * + */ + if (sock == INVALID_SOCKET) { + sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + } +#else + +#ifdef SOCK_CLOEXEC + auto sock = + socket(rp->ai_family, rp->ai_socktype | SOCK_CLOEXEC, rp->ai_protocol); +#else + auto sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); +#endif + +#endif + if (sock == INVALID_SOCKET) { continue; } + +#if !defined _WIN32 && !defined SOCK_CLOEXEC + if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) { + close_socket(sock); + continue; + } +#endif + + if (tcp_nodelay) { set_socket_opt(sock, IPPROTO_TCP, TCP_NODELAY, 1); } + + if (rp->ai_family == AF_INET6) { + set_socket_opt(sock, IPPROTO_IPV6, IPV6_V6ONLY, ipv6_v6only ? 1 : 0); + } + + if (socket_options) { socket_options(sock); } + + // bind or connect + auto quit = false; + if (bind_or_connect(sock, *rp, quit)) { return sock; } + + close_socket(sock); + + if (quit) { break; } + } + + return INVALID_SOCKET; +} + +inline void set_nonblocking(socket_t sock, bool nonblocking) { +#ifdef _WIN32 + auto flags = nonblocking ? 1UL : 0UL; + ioctlsocket(sock, FIONBIO, &flags); +#else + auto flags = fcntl(sock, F_GETFL, 0); + fcntl(sock, F_SETFL, + nonblocking ? (flags | O_NONBLOCK) : (flags & (~O_NONBLOCK))); +#endif +} + +inline bool is_connection_error() { +#ifdef _WIN32 + return WSAGetLastError() != WSAEWOULDBLOCK; +#else + return errno != EINPROGRESS; +#endif +} + +inline bool bind_ip_address(socket_t sock, const std::string &host) { + struct addrinfo hints; + struct addrinfo *result; + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = 0; + + if (getaddrinfo_with_timeout(host.c_str(), "0", &hints, &result, 0)) { + return false; + } + + auto se = detail::scope_exit([&] { freeaddrinfo(result); }); + + auto ret = false; + for (auto rp = result; rp; rp = rp->ai_next) { + const auto &ai = *rp; + if (!::bind(sock, ai.ai_addr, static_cast(ai.ai_addrlen))) { + ret = true; + break; + } + } + + return ret; +} + +#if !defined _WIN32 && !defined ANDROID && !defined _AIX && !defined __MVS__ +#define USE_IF2IP +#endif + +#ifdef USE_IF2IP +inline std::string if2ip(int address_family, const std::string &ifn) { + struct ifaddrs *ifap; + getifaddrs(&ifap); + auto se = detail::scope_exit([&] { freeifaddrs(ifap); }); + + std::string addr_candidate; + for (auto ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr && ifn == ifa->ifa_name && + (AF_UNSPEC == address_family || + ifa->ifa_addr->sa_family == address_family)) { + if (ifa->ifa_addr->sa_family == AF_INET) { + auto sa = reinterpret_cast(ifa->ifa_addr); + char buf[INET_ADDRSTRLEN]; + if (inet_ntop(AF_INET, &sa->sin_addr, buf, INET_ADDRSTRLEN)) { + return std::string(buf, INET_ADDRSTRLEN); + } + } else if (ifa->ifa_addr->sa_family == AF_INET6) { + auto sa = reinterpret_cast(ifa->ifa_addr); + if (!IN6_IS_ADDR_LINKLOCAL(&sa->sin6_addr)) { + char buf[INET6_ADDRSTRLEN] = {}; + if (inet_ntop(AF_INET6, &sa->sin6_addr, buf, INET6_ADDRSTRLEN)) { + // equivalent to mac's IN6_IS_ADDR_UNIQUE_LOCAL + auto s6_addr_head = sa->sin6_addr.s6_addr[0]; + if (s6_addr_head == 0xfc || s6_addr_head == 0xfd) { + addr_candidate = std::string(buf, INET6_ADDRSTRLEN); + } else { + return std::string(buf, INET6_ADDRSTRLEN); + } + } + } + } + } + } + return addr_candidate; +} +#endif + +inline socket_t create_client_socket( + const std::string &host, const std::string &ip, int port, + int address_family, bool tcp_nodelay, bool ipv6_v6only, + SocketOptions socket_options, time_t connection_timeout_sec, + time_t connection_timeout_usec, time_t read_timeout_sec, + time_t read_timeout_usec, time_t write_timeout_sec, + time_t write_timeout_usec, const std::string &intf, Error &error) { + auto sock = create_socket( + host, ip, port, address_family, 0, tcp_nodelay, ipv6_v6only, + std::move(socket_options), + [&](socket_t sock2, struct addrinfo &ai, bool &quit) -> bool { + if (!intf.empty()) { +#ifdef USE_IF2IP + auto ip_from_if = if2ip(address_family, intf); + if (ip_from_if.empty()) { ip_from_if = intf; } + if (!bind_ip_address(sock2, ip_from_if)) { + error = Error::BindIPAddress; + return false; + } +#endif + } + + set_nonblocking(sock2, true); + + auto ret = + ::connect(sock2, ai.ai_addr, static_cast(ai.ai_addrlen)); + + if (ret < 0) { + if (is_connection_error()) { + error = Error::Connection; + return false; + } + error = wait_until_socket_is_ready(sock2, connection_timeout_sec, + connection_timeout_usec); + if (error != Error::Success) { + if (error == Error::ConnectionTimeout) { quit = true; } + return false; + } + } + + set_nonblocking(sock2, false); + set_socket_opt_time(sock2, SOL_SOCKET, SO_RCVTIMEO, read_timeout_sec, + read_timeout_usec); + set_socket_opt_time(sock2, SOL_SOCKET, SO_SNDTIMEO, write_timeout_sec, + write_timeout_usec); + + error = Error::Success; + return true; + }, + connection_timeout_sec); // Pass DNS timeout + + if (sock != INVALID_SOCKET) { + error = Error::Success; + } else { + if (error == Error::Success) { error = Error::Connection; } + } + + return sock; +} + +inline bool get_ip_and_port(const struct sockaddr_storage &addr, + socklen_t addr_len, std::string &ip, int &port) { + if (addr.ss_family == AF_INET) { + port = ntohs(reinterpret_cast(&addr)->sin_port); + } else if (addr.ss_family == AF_INET6) { + port = + ntohs(reinterpret_cast(&addr)->sin6_port); + } else { + return false; + } + + std::array ipstr{}; + if (getnameinfo(reinterpret_cast(&addr), addr_len, + ipstr.data(), static_cast(ipstr.size()), nullptr, + 0, NI_NUMERICHOST)) { + return false; + } + + ip = ipstr.data(); + return true; +} + +inline void get_local_ip_and_port(socket_t sock, std::string &ip, int &port) { + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + if (!getsockname(sock, reinterpret_cast(&addr), + &addr_len)) { + get_ip_and_port(addr, addr_len, ip, port); + } +} + +inline void get_remote_ip_and_port(socket_t sock, std::string &ip, int &port) { + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + + if (!getpeername(sock, reinterpret_cast(&addr), + &addr_len)) { +#ifndef _WIN32 + if (addr.ss_family == AF_UNIX) { +#if defined(__linux__) + struct ucred ucred; + socklen_t len = sizeof(ucred); + if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &len) == 0) { + port = ucred.pid; + } +#elif defined(SOL_LOCAL) && defined(SO_PEERPID) + pid_t pid; + socklen_t len = sizeof(pid); + if (getsockopt(sock, SOL_LOCAL, SO_PEERPID, &pid, &len) == 0) { + port = pid; + } +#endif + return; + } +#endif + get_ip_and_port(addr, addr_len, ip, port); + } +} + +// Recursive form retained so operator""_t below can compute hashes for +// switch-case labels at compile time (C++11 constexpr forbids loops). Do not +// call from runtime paths with arbitrary-length inputs — use str2tag() +// instead, which is iterative and stack-safe. +inline constexpr unsigned int str2tag_core(const char *s, size_t l, + unsigned int h) { + return (l == 0) + ? h + : str2tag_core( + s + 1, l - 1, + // Unsets the 6 high bits of h, therefore no overflow happens + (((std::numeric_limits::max)() >> 6) & + h * 33) ^ + static_cast(*s)); +} + +inline unsigned int str2tag(const std::string &s) { + // Iterative form of str2tag_core: the recursive constexpr version is kept + // for compile-time UDL evaluation of short string literals, but at runtime + // we may receive arbitrarily long inputs (e.g. fuzzed Content-Type) that + // would blow the stack with one frame per character. + unsigned int h = 0; + for (auto c : s) { + h = (((std::numeric_limits::max)() >> 6) & h * 33) ^ + static_cast(c); + } + return h; +} + +namespace udl { + +inline constexpr unsigned int operator""_t(const char *s, size_t l) { + return str2tag_core(s, l, 0); +} + +} // namespace udl + +inline std::string +find_content_type(const std::string &path, + const std::map &user_data, + const std::string &default_content_type) { + auto ext = file_extension(path); + + auto it = user_data.find(ext); + if (it != user_data.end()) { return it->second; } + + using udl::operator""_t; + + switch (str2tag(ext)) { + default: return default_content_type; + + case "css"_t: return "text/css"; + case "csv"_t: return "text/csv"; + case "htm"_t: + case "html"_t: return "text/html"; + case "js"_t: + case "mjs"_t: return "text/javascript"; + case "txt"_t: return "text/plain"; + case "vtt"_t: return "text/vtt"; + + case "apng"_t: return "image/apng"; + case "avif"_t: return "image/avif"; + case "bmp"_t: return "image/bmp"; + case "gif"_t: return "image/gif"; + case "png"_t: return "image/png"; + case "svg"_t: return "image/svg+xml"; + case "webp"_t: return "image/webp"; + case "ico"_t: return "image/x-icon"; + case "tif"_t: return "image/tiff"; + case "tiff"_t: return "image/tiff"; + case "jpg"_t: + case "jpeg"_t: return "image/jpeg"; + + case "mp4"_t: return "video/mp4"; + case "mpeg"_t: return "video/mpeg"; + case "webm"_t: return "video/webm"; + + case "mp3"_t: return "audio/mp3"; + case "mpga"_t: return "audio/mpeg"; + case "weba"_t: return "audio/webm"; + case "wav"_t: return "audio/wave"; + + case "otf"_t: return "font/otf"; + case "ttf"_t: return "font/ttf"; + case "woff"_t: return "font/woff"; + case "woff2"_t: return "font/woff2"; + + case "7z"_t: return "application/x-7z-compressed"; + case "atom"_t: return "application/atom+xml"; + case "pdf"_t: return "application/pdf"; + case "json"_t: return "application/json"; + case "rss"_t: return "application/rss+xml"; + case "tar"_t: return "application/x-tar"; + case "xht"_t: + case "xhtml"_t: return "application/xhtml+xml"; + case "xslt"_t: return "application/xslt+xml"; + case "xml"_t: return "application/xml"; + case "gz"_t: return "application/gzip"; + case "zip"_t: return "application/zip"; + case "wasm"_t: return "application/wasm"; + } +} + +inline std::string +extract_media_type(const std::string &content_type, + std::map *params = nullptr) { + // Extract type/subtype from Content-Type value (RFC 2045) + // e.g. "application/json; charset=utf-8" -> "application/json" + auto media_type = content_type; + auto semicolon_pos = media_type.find(';'); + if (semicolon_pos != std::string::npos) { + auto param_str = media_type.substr(semicolon_pos + 1); + media_type = media_type.substr(0, semicolon_pos); + + if (params) { + // Parse parameters: key=value pairs separated by ';' + split(param_str.data(), param_str.data() + param_str.size(), ';', + [&](const char *b, const char *e) { + std::string key; + std::string val; + split(b, e, '=', [&](const char *b2, const char *e2) { + if (key.empty()) { + key.assign(b2, e2); + } else { + val.assign(b2, e2); + } + }); + if (!key.empty()) { + params->emplace(trim_copy(key), trim_double_quotes_copy(val)); + } + }); + } + } + + // Trim whitespace from media type + return trim_copy(media_type); +} + +inline bool can_compress_content_type(const std::string &content_type) { + using udl::operator""_t; + + auto mime_type = extract_media_type(content_type); + auto tag = str2tag(mime_type); + + switch (tag) { + case "image/svg+xml"_t: + case "application/javascript"_t: + case "application/x-javascript"_t: + case "application/json"_t: + case "application/ld+json"_t: + case "application/xml"_t: + case "application/xhtml+xml"_t: + case "application/rss+xml"_t: + case "application/atom+xml"_t: + case "application/xslt+xml"_t: + case "application/protobuf"_t: return true; + + case "text/event-stream"_t: return false; + + default: return !mime_type.rfind("text/", 0); + } +} + +inline bool parse_quality(const char *b, const char *e, std::string &token, + double &quality) { + quality = 1.0; + token.clear(); + + // Split on first ';': left = token name, right = parameters + const char *params_b = nullptr; + std::size_t params_len = 0; + + divide( + b, static_cast(e - b), ';', + [&](const char *lb, std::size_t llen, const char *rb, std::size_t rlen) { + auto r = trim(lb, lb + llen, 0, llen); + if (r.first < r.second) { token.assign(lb + r.first, lb + r.second); } + params_b = rb; + params_len = rlen; + }); + + if (token.empty()) { return false; } + if (params_len == 0) { return true; } + + // Scan parameters for q= (stops on first match) + bool invalid = false; + split_find(params_b, params_b + params_len, ';', + (std::numeric_limits::max)(), + [&](const char *pb, const char *pe) -> bool { + // Match exactly "q=" or "Q=" (not "query=" etc.) + auto len = static_cast(pe - pb); + if (len < 2) { return false; } + if ((pb[0] != 'q' && pb[0] != 'Q') || pb[1] != '=') { + return false; + } + + // Trim the value portion + auto r = trim(pb, pe, 2, len); + if (r.first >= r.second) { + invalid = true; + return true; + } + + double v = 0.0; + auto res = from_chars(pb + r.first, pb + r.second, v); + if (res.ec != std::errc{} || v < 0.0 || v > 1.0) { + invalid = true; + return true; + } + quality = v; + return true; + }); + + return !invalid; +} + +inline EncodingType encoding_type(const Request &req, const Response &res) { + if (!can_compress_content_type(res.get_header_value("Content-Type"))) { + return EncodingType::None; + } + + const auto &s = req.get_header_value("Accept-Encoding"); + if (s.empty()) { return EncodingType::None; } + + // Single-pass: iterate tokens and track the best supported encoding. + // Server preference breaks ties (br > gzip > zstd). + EncodingType best = EncodingType::None; + double best_q = 0.0; // q=0 means "not acceptable" + + // Server preference: Brotli > Gzip > Zstd (lower = more preferred) + auto priority = [](EncodingType t) -> int { + switch (t) { + case EncodingType::Brotli: return 0; + case EncodingType::Gzip: return 1; + case EncodingType::Zstd: return 2; + default: return 3; + } + }; + + std::string name; + split(s.data(), s.data() + s.size(), ',', [&](const char *b, const char *e) { + double quality = 1.0; + if (!parse_quality(b, e, name, quality)) { return; } + if (quality <= 0.0) { return; } + + EncodingType type = EncodingType::None; +#ifdef CPPHTTPLIB_BROTLI_SUPPORT + if (case_ignore::equal(name, "br")) { type = EncodingType::Brotli; } +#endif +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + if (type == EncodingType::None && case_ignore::equal(name, "gzip")) { + type = EncodingType::Gzip; + } +#endif +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + if (type == EncodingType::None && case_ignore::equal(name, "zstd")) { + type = EncodingType::Zstd; + } +#endif + + if (type == EncodingType::None) { return; } + + // Higher q-value wins; for equal q, server preference breaks ties + if (quality > best_q || + (quality == best_q && priority(type) < priority(best))) { + best_q = quality; + best = type; + } + }); + + return best; +} + +inline std::unique_ptr make_compressor(EncodingType type) { +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + if (type == EncodingType::Gzip) { + return detail::make_unique(); + } +#endif +#ifdef CPPHTTPLIB_BROTLI_SUPPORT + if (type == EncodingType::Brotli) { + return detail::make_unique(); + } +#endif +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + if (type == EncodingType::Zstd) { + return detail::make_unique(); + } +#endif + (void)type; + return nullptr; +} + +inline const char *encoding_name(EncodingType type) { + switch (type) { + case EncodingType::Gzip: return "gzip"; + case EncodingType::Brotli: return "br"; + case EncodingType::Zstd: return "zstd"; + default: return ""; + } +} + +inline bool nocompressor::compress(const char *data, size_t data_length, + bool /*last*/, Callback callback) { + if (!data_length) { return true; } + return callback(data, data_length); +} + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT +inline gzip_compressor::gzip_compressor() { + std::memset(&strm_, 0, sizeof(strm_)); + strm_.zalloc = Z_NULL; + strm_.zfree = Z_NULL; + strm_.opaque = Z_NULL; + + is_valid_ = deflateInit2(&strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8, + Z_DEFAULT_STRATEGY) == Z_OK; +} + +inline gzip_compressor::~gzip_compressor() { deflateEnd(&strm_); } + +inline bool gzip_compressor::compress(const char *data, size_t data_length, + bool last, Callback callback) { + assert(is_valid_); + + do { + constexpr size_t max_avail_in = + (std::numeric_limits::max)(); + + strm_.avail_in = static_cast( + (std::min)(data_length, max_avail_in)); + strm_.next_in = const_cast(reinterpret_cast(data)); + + data_length -= strm_.avail_in; + data += strm_.avail_in; + + auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH; + auto ret = Z_OK; + + std::array buff{}; + do { + strm_.avail_out = static_cast(buff.size()); + strm_.next_out = reinterpret_cast(buff.data()); + + ret = deflate(&strm_, flush); + if (ret == Z_STREAM_ERROR) { return false; } + + if (!callback(buff.data(), buff.size() - strm_.avail_out)) { + return false; + } + } while (strm_.avail_out == 0); + + assert((flush == Z_FINISH && ret == Z_STREAM_END) || + (flush == Z_NO_FLUSH && ret == Z_OK)); + assert(strm_.avail_in == 0); + } while (data_length > 0); + + return true; +} + +inline gzip_decompressor::gzip_decompressor() { + std::memset(&strm_, 0, sizeof(strm_)); + strm_.zalloc = Z_NULL; + strm_.zfree = Z_NULL; + strm_.opaque = Z_NULL; + + // 15 is the value of wbits, which should be at the maximum possible value + // to ensure that any gzip stream can be decoded. The offset of 32 specifies + // that the stream type should be automatically detected either gzip or + // deflate. + is_valid_ = inflateInit2(&strm_, 32 + 15) == Z_OK; +} + +inline gzip_decompressor::~gzip_decompressor() { inflateEnd(&strm_); } + +inline bool gzip_decompressor::is_valid() const { return is_valid_; } + +inline bool gzip_decompressor::decompress(const char *data, size_t data_length, + Callback callback) { + assert(is_valid_); + + auto ret = Z_OK; + + do { + constexpr size_t max_avail_in = + (std::numeric_limits::max)(); + + strm_.avail_in = static_cast( + (std::min)(data_length, max_avail_in)); + strm_.next_in = const_cast(reinterpret_cast(data)); + + data_length -= strm_.avail_in; + data += strm_.avail_in; + + std::array buff{}; + while (strm_.avail_in > 0 && ret == Z_OK) { + strm_.avail_out = static_cast(buff.size()); + strm_.next_out = reinterpret_cast(buff.data()); + + ret = inflate(&strm_, Z_NO_FLUSH); + + assert(ret != Z_STREAM_ERROR); + switch (ret) { + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_MEM_ERROR: inflateEnd(&strm_); return false; + } + + if (!callback(buff.data(), buff.size() - strm_.avail_out)) { + return false; + } + } + + if (ret != Z_OK && ret != Z_STREAM_END) { return false; } + + } while (data_length > 0); + + return true; +} +#endif + +#ifdef CPPHTTPLIB_BROTLI_SUPPORT +inline brotli_compressor::brotli_compressor() { + state_ = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); +} + +inline brotli_compressor::~brotli_compressor() { + BrotliEncoderDestroyInstance(state_); +} + +inline bool brotli_compressor::compress(const char *data, size_t data_length, + bool last, Callback callback) { + std::array buff{}; + + auto operation = last ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS; + auto available_in = data_length; + auto next_in = reinterpret_cast(data); + + for (;;) { + if (last) { + if (BrotliEncoderIsFinished(state_)) { break; } + } else { + if (!available_in) { break; } + } + + auto available_out = buff.size(); + auto next_out = buff.data(); + + if (!BrotliEncoderCompressStream(state_, operation, &available_in, &next_in, + &available_out, &next_out, nullptr)) { + return false; + } + + auto output_bytes = buff.size() - available_out; + if (output_bytes) { + callback(reinterpret_cast(buff.data()), output_bytes); + } + } + + return true; +} + +inline brotli_decompressor::brotli_decompressor() { + decoder_s = BrotliDecoderCreateInstance(0, 0, 0); + decoder_r = decoder_s ? BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT + : BROTLI_DECODER_RESULT_ERROR; +} + +inline brotli_decompressor::~brotli_decompressor() { + if (decoder_s) { BrotliDecoderDestroyInstance(decoder_s); } +} + +inline bool brotli_decompressor::is_valid() const { return decoder_s; } + +inline bool brotli_decompressor::decompress(const char *data, + size_t data_length, + Callback callback) { + if (decoder_r == BROTLI_DECODER_RESULT_SUCCESS || + decoder_r == BROTLI_DECODER_RESULT_ERROR) { + return 0; + } + + auto next_in = reinterpret_cast(data); + size_t avail_in = data_length; + size_t total_out; + + decoder_r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; + + std::array buff{}; + while (decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + char *next_out = buff.data(); + size_t avail_out = buff.size(); + + decoder_r = BrotliDecoderDecompressStream( + decoder_s, &avail_in, &next_in, &avail_out, + reinterpret_cast(&next_out), &total_out); + + if (decoder_r == BROTLI_DECODER_RESULT_ERROR) { return false; } + + if (!callback(buff.data(), buff.size() - avail_out)) { return false; } + } + + return decoder_r == BROTLI_DECODER_RESULT_SUCCESS || + decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT; +} +#endif + +#ifdef CPPHTTPLIB_ZSTD_SUPPORT +inline zstd_compressor::zstd_compressor() { + ctx_ = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(ctx_, ZSTD_c_compressionLevel, ZSTD_fast); +} + +inline zstd_compressor::~zstd_compressor() { ZSTD_freeCCtx(ctx_); } + +inline bool zstd_compressor::compress(const char *data, size_t data_length, + bool last, Callback callback) { + std::array buff{}; + + ZSTD_EndDirective mode = last ? ZSTD_e_end : ZSTD_e_continue; + ZSTD_inBuffer input = {data, data_length, 0}; + + bool finished; + do { + ZSTD_outBuffer output = {buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0}; + size_t const remaining = ZSTD_compressStream2(ctx_, &output, &input, mode); + + if (ZSTD_isError(remaining)) { return false; } + + if (!callback(buff.data(), output.pos)) { return false; } + + finished = last ? (remaining == 0) : (input.pos == input.size); + + } while (!finished); + + return true; +} + +inline zstd_decompressor::zstd_decompressor() { ctx_ = ZSTD_createDCtx(); } + +inline zstd_decompressor::~zstd_decompressor() { ZSTD_freeDCtx(ctx_); } + +inline bool zstd_decompressor::is_valid() const { return ctx_ != nullptr; } + +inline bool zstd_decompressor::decompress(const char *data, size_t data_length, + Callback callback) { + std::array buff{}; + ZSTD_inBuffer input = {data, data_length, 0}; + + while (input.pos < input.size) { + ZSTD_outBuffer output = {buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0}; + size_t const remaining = ZSTD_decompressStream(ctx_, &output, &input); + + if (ZSTD_isError(remaining)) { return false; } + + if (!callback(buff.data(), output.pos)) { return false; } + } + + return true; +} +#endif + +inline std::unique_ptr +create_decompressor(const std::string &encoding) { + std::unique_ptr decompressor; + + if (encoding == "gzip" || encoding == "deflate") { +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + decompressor = detail::make_unique(); +#endif + } else if (encoding.find("br") != std::string::npos) { +#ifdef CPPHTTPLIB_BROTLI_SUPPORT + decompressor = detail::make_unique(); +#endif + } else if (encoding == "zstd" || encoding.find("zstd") != std::string::npos) { +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + decompressor = detail::make_unique(); +#endif + } + + return decompressor; +} + +// Returns the best available compressor and its Content-Encoding name. +// Priority: Brotli > Gzip > Zstd (matches server-side preference). +inline std::pair, const char *> +create_compressor() { +#ifdef CPPHTTPLIB_BROTLI_SUPPORT + return {detail::make_unique(), "br"}; +#elif defined(CPPHTTPLIB_ZLIB_SUPPORT) + return {detail::make_unique(), "gzip"}; +#elif defined(CPPHTTPLIB_ZSTD_SUPPORT) + return {detail::make_unique(), "zstd"}; +#else + return {nullptr, nullptr}; +#endif +} + +inline bool is_prohibited_header_name(const std::string &name) { + using udl::operator""_t; + + switch (str2tag(name)) { + case "REMOTE_ADDR"_t: + case "REMOTE_PORT"_t: + case "LOCAL_ADDR"_t: + case "LOCAL_PORT"_t: return true; + default: return false; + } +} + +inline bool has_header(const Headers &headers, const std::string &key) { + if (is_prohibited_header_name(key)) { return false; } + return headers.find(key) != headers.end(); +} + +inline const char *get_header_value(const Headers &headers, + const std::string &key, const char *def, + size_t id) { + if (is_prohibited_header_name(key)) { +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + std::string msg = "Prohibited header name '" + key + "' is specified."; + throw std::invalid_argument(msg); +#else + return ""; +#endif + } + + auto rng = headers.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { return it->second.c_str(); } + return def; +} + +inline size_t get_header_value_count(const Headers &headers, + const std::string &key) { + auto r = headers.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +template +inline typename Map::mapped_type +get_multimap_value(const Map &m, const std::string &key, size_t id) { + auto rng = m.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { return it->second; } + return typename Map::mapped_type(); +} + +inline void set_header(Headers &headers, const std::string &key, + const std::string &val) { + if (fields::is_field_name(key) && fields::is_field_value(val)) { + headers.emplace(key, val); + } +} + +inline bool read_headers(Stream &strm, Headers &headers) { + const auto bufsiz = 2048; + char buf[bufsiz]; + stream_line_reader line_reader(strm, buf, bufsiz); + + size_t header_count = 0; + + for (;;) { + if (!line_reader.getline()) { return false; } + + // Check if the line ends with CRLF. + auto line_terminator_len = 2; + if (line_reader.end_with_crlf()) { + // Blank line indicates end of headers. + if (line_reader.size() == 2) { break; } + } else { +#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR + // Blank line indicates end of headers. + if (line_reader.size() == 1) { break; } + line_terminator_len = 1; +#else + continue; // Skip invalid line. +#endif + } + + if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; } + + // Check header count limit + if (header_count >= CPPHTTPLIB_HEADER_MAX_COUNT) { return false; } + + // Exclude line terminator + auto end = line_reader.ptr() + line_reader.size() - line_terminator_len; + + if (!parse_header(line_reader.ptr(), end, + [&](const std::string &key, const std::string &val) { + headers.emplace(key, val); + })) { + return false; + } + + header_count++; + } + + // RFC 9110 Section 8.6: Reject requests with multiple Content-Length + // headers that have different values to prevent request smuggling. + auto cl_range = headers.equal_range("Content-Length"); + if (cl_range.first != cl_range.second) { + const auto &first_val = cl_range.first->second; + for (auto it = std::next(cl_range.first); it != cl_range.second; ++it) { + if (it->second != first_val) { return false; } + } + } + + return true; +} + +inline bool read_websocket_upgrade_response(Stream &strm, + const std::string &expected_accept, + std::string &selected_subprotocol) { + // Read status line + const auto bufsiz = 2048; + char buf[bufsiz]; + stream_line_reader line_reader(strm, buf, bufsiz); + if (!line_reader.getline()) { return false; } + + // Check for "HTTP/1.1 101" + auto line = std::string(line_reader.ptr(), line_reader.size()); + if (line.find("HTTP/1.1 101") == std::string::npos) { return false; } + + // Parse headers using existing read_headers + Headers headers; + if (!read_headers(strm, headers)) { return false; } + + // Verify Upgrade: websocket (case-insensitive) + auto upgrade_it = headers.find("Upgrade"); + if (upgrade_it == headers.end()) { return false; } + auto upgrade_val = case_ignore::to_lower(upgrade_it->second); + if (upgrade_val != "websocket") { return false; } + + // Verify Connection header contains "Upgrade" (case-insensitive) + auto connection_it = headers.find("Connection"); + if (connection_it == headers.end()) { return false; } + auto connection_val = case_ignore::to_lower(connection_it->second); + if (connection_val.find("upgrade") == std::string::npos) { return false; } + + // Verify Sec-WebSocket-Accept header value + auto it = headers.find("Sec-WebSocket-Accept"); + if (it == headers.end() || it->second != expected_accept) { return false; } + + // Extract negotiated subprotocol + auto proto_it = headers.find("Sec-WebSocket-Protocol"); + if (proto_it != headers.end()) { selected_subprotocol = proto_it->second; } + + return true; +} + +enum class ReadContentResult { + Success, // Successfully read the content + PayloadTooLarge, // The content exceeds the specified payload limit + Error // An error occurred while reading the content +}; + +inline ReadContentResult read_content_with_length( + Stream &strm, size_t len, DownloadProgress progress, + ContentReceiverWithProgress out, + size_t payload_max_length = (std::numeric_limits::max)()) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + + detail::BodyReader br; + br.stream = &strm; + br.has_content_length = true; + br.content_length = len; + br.payload_max_length = payload_max_length; + br.chunked = false; + br.bytes_read = 0; + br.last_error = Error::Success; + + size_t r = 0; + while (r < len) { + auto read_len = static_cast(len - r); + auto to_read = (std::min)(read_len, CPPHTTPLIB_RECV_BUFSIZ); + auto n = detail::read_body_content(&strm, br, buf, to_read); + if (n <= 0) { + // Check if it was a payload size error + if (br.last_error == Error::ExceedMaxPayloadSize) { + return ReadContentResult::PayloadTooLarge; + } + return ReadContentResult::Error; + } + + if (!out(buf, static_cast(n), r, len)) { + return ReadContentResult::Error; + } + r += static_cast(n); + + if (progress) { + if (!progress(r, len)) { return ReadContentResult::Error; } + } + } + + return ReadContentResult::Success; +} + +inline ReadContentResult +read_content_without_length(Stream &strm, size_t payload_max_length, + ContentReceiverWithProgress out) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + size_t r = 0; + for (;;) { + auto n = strm.read(buf, CPPHTTPLIB_RECV_BUFSIZ); + if (n == 0) { return ReadContentResult::Success; } + if (n < 0) { return ReadContentResult::Error; } + + // Check if adding this data would exceed the payload limit + if (r > payload_max_length || + payload_max_length - r < static_cast(n)) { + return ReadContentResult::PayloadTooLarge; + } + + if (!out(buf, static_cast(n), r, 0)) { + return ReadContentResult::Error; + } + r += static_cast(n); + } + + return ReadContentResult::Success; +} + +template +inline ReadContentResult read_content_chunked(Stream &strm, T &x, + size_t payload_max_length, + ContentReceiverWithProgress out) { + detail::ChunkedDecoder dec(strm); + + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + size_t total_len = 0; + + for (;;) { + size_t chunk_offset = 0; + size_t chunk_total = 0; + auto n = dec.read_payload(buf, sizeof(buf), chunk_offset, chunk_total); + if (n < 0) { return ReadContentResult::Error; } + + if (n == 0) { + if (!dec.parse_trailers_into(x.trailers, x.headers)) { + return ReadContentResult::Error; + } + return ReadContentResult::Success; + } + + if (total_len > payload_max_length || + payload_max_length - total_len < static_cast(n)) { + return ReadContentResult::PayloadTooLarge; + } + + if (!out(buf, static_cast(n), chunk_offset, chunk_total)) { + return ReadContentResult::Error; + } + + total_len += static_cast(n); + } +} + +inline bool is_chunked_transfer_encoding(const Headers &headers) { + return case_ignore::equal( + get_header_value(headers, "Transfer-Encoding", "", 0), "chunked"); +} + +template +bool prepare_content_receiver(T &x, int &status, + ContentReceiverWithProgress receiver, + bool decompress, size_t payload_max_length, + bool &exceed_payload_max_length, U callback) { + if (decompress) { + std::string encoding = x.get_header_value("Content-Encoding"); + std::unique_ptr decompressor; + + if (!encoding.empty()) { + decompressor = detail::create_decompressor(encoding); + if (!decompressor) { + // Unsupported encoding or no support compiled in + status = StatusCode::UnsupportedMediaType_415; + return false; + } + } + + if (decompressor) { + if (decompressor->is_valid()) { + size_t decompressed_size = 0; + ContentReceiverWithProgress out = [&](const char *buf, size_t n, + size_t off, size_t len) { + return decompressor->decompress( + buf, n, [&](const char *buf2, size_t n2) { + // Guard against zip-bomb: check + // decompressed size against limit. + if (payload_max_length > 0 && + (decompressed_size >= payload_max_length || + n2 > payload_max_length - decompressed_size)) { + exceed_payload_max_length = true; + return false; + } + decompressed_size += n2; + return receiver(buf2, n2, off, len); + }); + }; + return callback(std::move(out)); + } else { + status = StatusCode::InternalServerError_500; + return false; + } + } + } + + ContentReceiverWithProgress out = [&](const char *buf, size_t n, size_t off, + size_t len) { + return receiver(buf, n, off, len); + }; + return callback(std::move(out)); +} + +template +bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status, + DownloadProgress progress, + ContentReceiverWithProgress receiver, bool decompress) { + bool exceed_payload_max_length = false; + return prepare_content_receiver( + x, status, std::move(receiver), decompress, payload_max_length, + exceed_payload_max_length, [&](const ContentReceiverWithProgress &out) { + auto ret = true; + // Note: exceed_payload_max_length may also be set by the decompressor + // wrapper in prepare_content_receiver when the decompressed payload + // size exceeds the limit. + + if (is_chunked_transfer_encoding(x.headers)) { + auto result = read_content_chunked(strm, x, payload_max_length, out); + if (result == ReadContentResult::Success) { + ret = true; + } else if (result == ReadContentResult::PayloadTooLarge) { + exceed_payload_max_length = true; + ret = false; + } else { + ret = false; + } + } else if (!has_header(x.headers, "Content-Length")) { + auto result = + read_content_without_length(strm, payload_max_length, out); + if (result == ReadContentResult::Success) { + ret = true; + } else if (result == ReadContentResult::PayloadTooLarge) { + exceed_payload_max_length = true; + ret = false; + } else { + ret = false; + } + } else { + auto is_invalid_value = false; + auto len = get_header_value_u64(x.headers, "Content-Length", + (std::numeric_limits::max)(), + 0, is_invalid_value); + + if (is_invalid_value) { + ret = false; + } else if (len > 0) { + auto result = read_content_with_length( + strm, len, std::move(progress), out, payload_max_length); + ret = (result == ReadContentResult::Success); + if (result == ReadContentResult::PayloadTooLarge) { + exceed_payload_max_length = true; + } + } + } + + if (!ret) { + status = exceed_payload_max_length ? StatusCode::PayloadTooLarge_413 + : StatusCode::BadRequest_400; + } + return ret; + }); +} + +inline ssize_t write_request_line(Stream &strm, const std::string &method, + const std::string &path) { + std::string s = method; + s += ' '; + s += path; + s += " HTTP/1.1\r\n"; + return strm.write(s.data(), s.size()); +} + +inline ssize_t write_response_line(Stream &strm, int status) { + std::string s = "HTTP/1.1 "; + s += std::to_string(status); + s += ' '; + s += httplib::status_message(status); + s += "\r\n"; + return strm.write(s.data(), s.size()); +} + +inline ssize_t write_headers(Stream &strm, const Headers &headers) { + ssize_t write_len = 0; + for (const auto &x : headers) { + std::string s; + s = x.first; + s += ": "; + s += x.second; + s += "\r\n"; + + auto len = strm.write(s.data(), s.size()); + if (len < 0) { return len; } + write_len += len; + } + auto len = strm.write("\r\n"); + if (len < 0) { return len; } + write_len += len; + return write_len; +} + +inline bool write_data(Stream &strm, const char *d, size_t l) { + size_t offset = 0; + while (offset < l) { + auto length = strm.write(d + offset, l - offset); + if (length < 0) { return false; } + offset += static_cast(length); + } + return true; +} + +template +inline bool write_content_with_progress(Stream &strm, + const ContentProvider &content_provider, + size_t offset, size_t length, + T is_shutting_down, + const UploadProgress &upload_progress, + Error &error) { + size_t end_offset = offset + length; + size_t start_offset = offset; + auto ok = true; + DataSink data_sink; + + data_sink.write = [&](const char *d, size_t l) -> bool { + if (ok) { + if (write_data(strm, d, l)) { + offset += l; + + if (upload_progress && length > 0) { + size_t current_written = offset - start_offset; + if (!upload_progress(current_written, length)) { + ok = false; + return false; + } + } + } else { + ok = false; + } + } + return ok; + }; + + data_sink.is_writable = [&]() -> bool { return strm.is_peer_alive(); }; + + while (offset < end_offset && !is_shutting_down()) { + if (!strm.wait_writable() || !strm.is_peer_alive()) { + error = Error::Write; + return false; + } else if (!content_provider(offset, end_offset - offset, data_sink)) { + error = Error::Canceled; + return false; + } else if (!ok) { + error = Error::Write; + return false; + } + } + + if (offset < end_offset) { // exited due to is_shutting_down(), not completion + error = Error::Write; + return false; + } + + error = Error::Success; + return true; +} + +template +inline bool write_content(Stream &strm, const ContentProvider &content_provider, + size_t offset, size_t length, T is_shutting_down, + Error &error) { + return write_content_with_progress(strm, content_provider, offset, length, + is_shutting_down, nullptr, error); +} + +template +inline bool write_content(Stream &strm, const ContentProvider &content_provider, + size_t offset, size_t length, + const T &is_shutting_down) { + auto error = Error::Success; + return write_content(strm, content_provider, offset, length, is_shutting_down, + error); +} + +template +inline bool +write_content_without_length(Stream &strm, + const ContentProvider &content_provider, + const T &is_shutting_down) { + size_t offset = 0; + auto data_available = true; + auto ok = true; + DataSink data_sink; + + data_sink.write = [&](const char *d, size_t l) -> bool { + if (ok) { + offset += l; + if (!write_data(strm, d, l)) { ok = false; } + } + return ok; + }; + + data_sink.is_writable = [&]() -> bool { return strm.is_peer_alive(); }; + + data_sink.done = [&](void) { data_available = false; }; + + while (data_available && !is_shutting_down()) { + if (!strm.wait_writable() || !strm.is_peer_alive()) { + return false; + } else if (!content_provider(offset, 0, data_sink)) { + return false; + } else if (!ok) { + return false; + } + } + return !data_available; // true only if done() was called, false if shutting + // down +} + +template +inline bool +write_content_chunked(Stream &strm, const ContentProvider &content_provider, + const T &is_shutting_down, U &compressor, Error &error) { + size_t offset = 0; + auto data_available = true; + auto ok = true; + DataSink data_sink; + + data_sink.write = [&](const char *d, size_t l) -> bool { + if (ok) { + data_available = l > 0; + offset += l; + + std::string payload; + if (compressor.compress(d, l, false, + [&](const char *data, size_t data_len) { + payload.append(data, data_len); + return true; + })) { + if (!payload.empty()) { + // Emit chunked response header and footer for each chunk + auto chunk = + from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n"; + if (!write_data(strm, chunk.data(), chunk.size())) { ok = false; } + } + } else { + ok = false; + } + } + return ok; + }; + + data_sink.is_writable = [&]() -> bool { return strm.is_peer_alive(); }; + + auto done_with_trailer = [&](const Headers *trailer) { + if (!ok) { return; } + + data_available = false; + + std::string payload; + if (!compressor.compress(nullptr, 0, true, + [&](const char *data, size_t data_len) { + payload.append(data, data_len); + return true; + })) { + ok = false; + return; + } + + if (!payload.empty()) { + // Emit chunked response header and footer for each chunk + auto chunk = from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n"; + if (!write_data(strm, chunk.data(), chunk.size())) { + ok = false; + return; + } + } + + constexpr const char done_marker[] = "0\r\n"; + if (!write_data(strm, done_marker, str_len(done_marker))) { ok = false; } + + // Trailer + if (trailer) { + for (const auto &kv : *trailer) { + std::string field_line = kv.first + ": " + kv.second + "\r\n"; + if (!write_data(strm, field_line.data(), field_line.size())) { + ok = false; + } + } + } + + constexpr const char crlf[] = "\r\n"; + if (!write_data(strm, crlf, str_len(crlf))) { ok = false; } + }; + + data_sink.done = [&](void) { done_with_trailer(nullptr); }; + + data_sink.done_with_trailer = [&](const Headers &trailer) { + done_with_trailer(&trailer); + }; + + while (data_available && !is_shutting_down()) { + if (!strm.wait_writable() || !strm.is_peer_alive()) { + error = Error::Write; + return false; + } else if (!content_provider(offset, 0, data_sink)) { + error = Error::Canceled; + return false; + } else if (!ok) { + error = Error::Write; + return false; + } + } + + if (data_available) { // exited due to is_shutting_down(), not done() + error = Error::Write; + return false; + } + + error = Error::Success; + return true; +} + +template +inline bool write_content_chunked(Stream &strm, + const ContentProvider &content_provider, + const T &is_shutting_down, U &compressor) { + auto error = Error::Success; + return write_content_chunked(strm, content_provider, is_shutting_down, + compressor, error); +} + +template +inline bool redirect(T &cli, Request &req, Response &res, + const std::string &path, const std::string &location, + Error &error) { + Request new_req = req; + new_req.path = path; + new_req.redirect_count_ -= 1; + + if (res.status == StatusCode::SeeOther_303 && + (req.method != "GET" && req.method != "HEAD")) { + new_req.method = "GET"; + new_req.body.clear(); + new_req.headers.clear(); + } + + Response new_res; + + auto ret = cli.send(new_req, new_res, error); + if (ret) { + req = std::move(new_req); + res = std::move(new_res); + + if (res.location.empty()) { res.location = location; } + } + return ret; +} + +inline std::string params_to_query_str(const Params ¶ms) { + std::string query; + + for (auto it = params.begin(); it != params.end(); ++it) { + if (it != params.begin()) { query += '&'; } + query += encode_query_component(it->first); + query += '='; + query += encode_query_component(it->second); + } + return query; +} + +inline void parse_query_text(const char *data, std::size_t size, + Params ¶ms) { + std::set cache; + split(data, data + size, '&', [&](const char *b, const char *e) { + std::string kv(b, e); + if (cache.find(kv) != cache.end()) { return; } + cache.insert(std::move(kv)); + + std::string key; + std::string val; + divide(b, static_cast(e - b), '=', + [&](const char *lhs_data, std::size_t lhs_size, const char *rhs_data, + std::size_t rhs_size) { + key.assign(lhs_data, lhs_size); + val.assign(rhs_data, rhs_size); + }); + + if (!key.empty()) { + params.emplace(decode_query_component(key), decode_query_component(val)); + } + }); +} + +inline void parse_query_text(const std::string &s, Params ¶ms) { + parse_query_text(s.data(), s.size(), params); +} + +// Normalize a query string by decoding and re-encoding each key/value pair +// while preserving the original parameter order. This avoids double-encoding +// and ensures consistent encoding without reordering (unlike Params which +// uses std::multimap and sorts keys). +inline std::string normalize_query_string(const std::string &query) { + std::string result; + split(query.data(), query.data() + query.size(), '&', + [&](const char *b, const char *e) { + std::string key; + std::string val; + divide(b, static_cast(e - b), '=', + [&](const char *lhs_data, std::size_t lhs_size, + const char *rhs_data, std::size_t rhs_size) { + key.assign(lhs_data, lhs_size); + val.assign(rhs_data, rhs_size); + }); + + if (!key.empty()) { + auto dec_key = decode_query_component(key); + auto dec_val = decode_query_component(val); + + if (!result.empty()) { result += '&'; } + result += encode_query_component(dec_key); + if (!val.empty() || std::find(b, e, '=') != e) { + result += '='; + result += encode_query_component(dec_val); + } + } + }); + return result; +} + +inline bool parse_multipart_boundary(const std::string &content_type, + std::string &boundary) { + std::map params; + extract_media_type(content_type, ¶ms); + auto it = params.find("boundary"); + if (it == params.end()) { return false; } + boundary = it->second; + return !boundary.empty(); +} + +inline void parse_disposition_params(const std::string &s, Params ¶ms) { + std::set cache; + split(s.data(), s.data() + s.size(), ';', [&](const char *b, const char *e) { + std::string kv(b, e); + if (cache.find(kv) != cache.end()) { return; } + cache.insert(kv); + + std::string key; + std::string val; + split(b, e, '=', [&](const char *b2, const char *e2) { + if (key.empty()) { + key.assign(b2, e2); + } else { + val.assign(b2, e2); + } + }); + + if (!key.empty()) { + params.emplace(trim_double_quotes_copy((key)), + trim_double_quotes_copy((val))); + } + }); +} + +#ifdef CPPHTTPLIB_NO_EXCEPTIONS +inline bool parse_range_header(const std::string &s, Ranges &ranges) { +#else +inline bool parse_range_header(const std::string &s, Ranges &ranges) try { +#endif + auto is_valid = [](const std::string &str) { + return std::all_of(str.cbegin(), str.cend(), + [](unsigned char c) { return std::isdigit(c); }); + }; + + if (s.size() > 7 && s.compare(0, 6, "bytes=") == 0) { + const auto pos = static_cast(6); + const auto len = static_cast(s.size() - 6); + auto all_valid_ranges = true; + split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) { + if (!all_valid_ranges) { return; } + + const auto it = std::find(b, e, '-'); + if (it == e) { + all_valid_ranges = false; + return; + } + + const auto lhs = std::string(b, it); + const auto rhs = std::string(it + 1, e); + if (!is_valid(lhs) || !is_valid(rhs)) { + all_valid_ranges = false; + return; + } + + ssize_t first = -1; + if (!lhs.empty()) { + ssize_t v; + auto res = detail::from_chars(lhs.data(), lhs.data() + lhs.size(), v); + if (res.ec == std::errc{}) { first = v; } + } + + ssize_t last = -1; + if (!rhs.empty()) { + ssize_t v; + auto res = detail::from_chars(rhs.data(), rhs.data() + rhs.size(), v); + if (res.ec == std::errc{}) { last = v; } + } + + if ((first == -1 && last == -1) || + (first != -1 && last != -1 && first > last)) { + all_valid_ranges = false; + return; + } + + ranges.emplace_back(first, last); + }); + return all_valid_ranges && !ranges.empty(); + } + return false; +#ifdef CPPHTTPLIB_NO_EXCEPTIONS +} +#else +} catch (...) { return false; } +#endif + +inline bool parse_accept_header(const std::string &s, + std::vector &content_types) { + content_types.clear(); + + // Empty string is considered valid (no preference) + if (s.empty()) { return true; } + + // Check for invalid patterns: leading/trailing commas or consecutive commas + if (s.front() == ',' || s.back() == ',' || + s.find(",,") != std::string::npos) { + return false; + } + + struct AcceptEntry { + std::string media_type; + double quality; + int order; + }; + + std::vector entries; + int order = 0; + bool has_invalid_entry = false; + + // Split by comma and parse each entry + split(s.data(), s.data() + s.size(), ',', [&](const char *b, const char *e) { + std::string entry(b, e); + entry = trim_copy(entry); + + if (entry.empty()) { + has_invalid_entry = true; + return; + } + + AcceptEntry accept_entry; + accept_entry.order = order++; + + if (!parse_quality(entry.data(), entry.data() + entry.size(), + accept_entry.media_type, accept_entry.quality)) { + has_invalid_entry = true; + return; + } + + // Remove additional parameters from media type + accept_entry.media_type = extract_media_type(accept_entry.media_type); + + // Basic validation of media type format + if (accept_entry.media_type.empty()) { + has_invalid_entry = true; + return; + } + + // Check for basic media type format (should contain '/' or be '*') + if (accept_entry.media_type != "*" && + accept_entry.media_type.find('/') == std::string::npos) { + has_invalid_entry = true; + return; + } + + entries.push_back(std::move(accept_entry)); + }); + + // Return false if any invalid entry was found + if (has_invalid_entry) { return false; } + + // Sort by quality (descending), then by original order (ascending) + std::sort(entries.begin(), entries.end(), + [](const AcceptEntry &a, const AcceptEntry &b) { + if (a.quality != b.quality) { + return a.quality > b.quality; // Higher quality first + } + return a.order < b.order; // Earlier order first for same quality + }); + + // Extract sorted media types + content_types.reserve(entries.size()); + for (auto &entry : entries) { + content_types.push_back(std::move(entry.media_type)); + } + + return true; +} + +class FormDataParser { +public: + FormDataParser() = default; + + void set_boundary(std::string &&boundary) { + boundary_ = std::move(boundary); + dash_boundary_crlf_ = dash_ + boundary_ + crlf_; + crlf_dash_boundary_ = crlf_ + dash_ + boundary_; + } + + bool is_valid() const { return is_valid_; } + + bool parse(const char *buf, size_t n, const FormDataHeader &header_callback, + const ContentReceiver &content_callback) { + + buf_append(buf, n); + + while (buf_size() > 0) { + switch (state_) { + case 0: { // Initial boundary + auto pos = buf_find(dash_boundary_crlf_); + if (pos == buf_size()) { return true; } + buf_erase(pos + dash_boundary_crlf_.size()); + state_ = 1; + break; + } + case 1: { // New entry + clear_file_info(); + state_ = 2; + break; + } + case 2: { // Headers + auto pos = buf_find(crlf_); + if (pos > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; } + while (pos < buf_size()) { + // Empty line + if (pos == 0) { + if (!header_callback(file_)) { + is_valid_ = false; + return false; + } + buf_erase(crlf_.size()); + state_ = 3; + break; + } + + const auto header = buf_head(pos); + + if (!parse_header(header.data(), header.data() + header.size(), + [&](const std::string &, const std::string &) {})) { + is_valid_ = false; + return false; + } + + // Parse and emplace space trimmed headers into a map + if (!parse_header( + header.data(), header.data() + header.size(), + [&](const std::string &key, const std::string &val) { + file_.headers.emplace(key, val); + })) { + is_valid_ = false; + return false; + } + + constexpr const char header_content_type[] = "Content-Type:"; + + if (start_with_case_ignore(header, header_content_type)) { + file_.content_type = + trim_copy(header.substr(str_len(header_content_type))); + } else { + std::string disposition_params; + if (parse_content_disposition(header, disposition_params)) { + Params params; + parse_disposition_params(disposition_params, params); + + auto it = params.find("name"); + if (it != params.end()) { + file_.name = it->second; + } else { + is_valid_ = false; + return false; + } + + it = params.find("filename"); + if (it != params.end()) { file_.filename = it->second; } + + it = params.find("filename*"); + if (it != params.end()) { + // RFC 5987: only UTF-8 encoding is allowed + const auto &val = it->second; + constexpr const char utf8_prefix[] = "UTF-8''"; + constexpr size_t prefix_len = str_len(utf8_prefix); + if (val.size() > prefix_len && + start_with_case_ignore(val, utf8_prefix)) { + file_.filename = decode_path_component( + val.substr(prefix_len)); // override... + } else { + is_valid_ = false; + return false; + } + } + } + } + buf_erase(pos + crlf_.size()); + pos = buf_find(crlf_); + } + if (state_ != 3) { return true; } + break; + } + case 3: { // Body + if (crlf_dash_boundary_.size() > buf_size()) { return true; } + auto pos = buf_find(crlf_dash_boundary_); + if (pos < buf_size()) { + if (!content_callback(buf_data(), pos)) { + is_valid_ = false; + return false; + } + buf_erase(pos + crlf_dash_boundary_.size()); + state_ = 4; + } else { + auto len = buf_size() - crlf_dash_boundary_.size(); + if (len > 0) { + if (!content_callback(buf_data(), len)) { + is_valid_ = false; + return false; + } + buf_erase(len); + } + return true; + } + break; + } + case 4: { // Boundary + if (crlf_.size() > buf_size()) { return true; } + if (buf_start_with(crlf_)) { + buf_erase(crlf_.size()); + state_ = 1; + } else { + if (dash_.size() > buf_size()) { return true; } + if (buf_start_with(dash_)) { + buf_erase(dash_.size()); + is_valid_ = true; + buf_erase(buf_size()); // Remove epilogue + } else { + return true; + } + } + break; + } + } + } + + return true; + } + +private: + void clear_file_info() { + file_.name.clear(); + file_.filename.clear(); + file_.content_type.clear(); + file_.headers.clear(); + } + + bool start_with_case_ignore(const std::string &a, const char *b, + size_t offset = 0) const { + const auto b_len = strlen(b); + if (a.size() < offset + b_len) { return false; } + for (size_t i = 0; i < b_len; i++) { + if (case_ignore::to_lower(a[offset + i]) != case_ignore::to_lower(b[i])) { + return false; + } + } + return true; + } + + // Parses "Content-Disposition: form-data; " without std::regex. + // Returns true if header matches, with the params portion in `params_out`. + bool parse_content_disposition(const std::string &header, + std::string ¶ms_out) const { + constexpr const char prefix[] = "Content-Disposition:"; + constexpr size_t prefix_len = str_len(prefix); + + if (!start_with_case_ignore(header, prefix)) { return false; } + + // Skip whitespace after "Content-Disposition:" + auto pos = prefix_len; + while (pos < header.size() && (header[pos] == ' ' || header[pos] == '\t')) { + pos++; + } + + // Match "form-data;" (case-insensitive) + constexpr const char form_data[] = "form-data;"; + constexpr size_t form_data_len = str_len(form_data); + if (!start_with_case_ignore(header, form_data, pos)) { return false; } + pos += form_data_len; + + // Skip whitespace after "form-data;" + while (pos < header.size() && (header[pos] == ' ' || header[pos] == '\t')) { + pos++; + } + + params_out = header.substr(pos); + return true; + } + + const std::string dash_ = "--"; + const std::string crlf_ = "\r\n"; + std::string boundary_; + std::string dash_boundary_crlf_; + std::string crlf_dash_boundary_; + + size_t state_ = 0; + bool is_valid_ = false; + FormData file_; + + // Buffer + bool start_with(const std::string &a, size_t spos, size_t epos, + const std::string &b) const { + if (epos - spos < b.size()) { return false; } + for (size_t i = 0; i < b.size(); i++) { + if (a[i + spos] != b[i]) { return false; } + } + return true; + } + + size_t buf_size() const { return buf_epos_ - buf_spos_; } + + const char *buf_data() const { return &buf_[buf_spos_]; } + + std::string buf_head(size_t l) const { return buf_.substr(buf_spos_, l); } + + bool buf_start_with(const std::string &s) const { + return start_with(buf_, buf_spos_, buf_epos_, s); + } + + size_t buf_find(const std::string &s) const { + auto c = s.front(); + + size_t off = buf_spos_; + while (off < buf_epos_) { + auto pos = off; + while (true) { + if (pos == buf_epos_) { return buf_size(); } + if (buf_[pos] == c) { break; } + pos++; + } + + auto remaining_size = buf_epos_ - pos; + if (s.size() > remaining_size) { return buf_size(); } + + if (start_with(buf_, pos, buf_epos_, s)) { return pos - buf_spos_; } + + off = pos + 1; + } + + return buf_size(); + } + + void buf_append(const char *data, size_t n) { + auto remaining_size = buf_size(); + if (remaining_size > 0 && buf_spos_ > 0) { + for (size_t i = 0; i < remaining_size; i++) { + buf_[i] = buf_[buf_spos_ + i]; + } + } + buf_spos_ = 0; + buf_epos_ = remaining_size; + + if (remaining_size + n > buf_.size()) { buf_.resize(remaining_size + n); } + + for (size_t i = 0; i < n; i++) { + buf_[buf_epos_ + i] = data[i]; + } + buf_epos_ += n; + } + + void buf_erase(size_t size) { buf_spos_ += size; } + + std::string buf_; + size_t buf_spos_ = 0; + size_t buf_epos_ = 0; +}; + +inline std::string random_string(size_t length) { + constexpr const char data[] = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + + thread_local auto engine([]() { + // std::random_device might actually be deterministic on some + // platforms, but due to lack of support in the c++ standard library, + // doing better requires either some ugly hacks or breaking portability. + std::random_device seed_gen; + // Request 128 bits of entropy for initialization + std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(), seed_gen()}; + return std::mt19937(seed_sequence); + }()); + + std::string result; + for (size_t i = 0; i < length; i++) { + result += data[engine() % (sizeof(data) - 1)]; + } + return result; +} + +inline std::string make_multipart_data_boundary() { + return "--cpp-httplib-multipart-data-" + detail::random_string(16); +} + +inline bool is_multipart_boundary_chars_valid(const std::string &boundary) { + auto valid = true; + for (size_t i = 0; i < boundary.size(); i++) { + auto c = boundary[i]; + if (!std::isalnum(static_cast(c)) && c != '-' && c != '_') { + valid = false; + break; + } + } + return valid; +} + +template +inline std::string +serialize_multipart_formdata_item_begin(const T &item, + const std::string &boundary) { + std::string body = "--" + boundary + "\r\n"; + body += "Content-Disposition: form-data; name=\"" + item.name + "\""; + if (!item.filename.empty()) { + body += "; filename=\"" + item.filename + "\""; + } + body += "\r\n"; + if (!item.content_type.empty()) { + body += "Content-Type: " + item.content_type + "\r\n"; + } + body += "\r\n"; + + return body; +} + +inline std::string serialize_multipart_formdata_item_end() { return "\r\n"; } + +inline std::string +serialize_multipart_formdata_finish(const std::string &boundary) { + return "--" + boundary + "--\r\n"; +} + +inline std::string +serialize_multipart_formdata_get_content_type(const std::string &boundary) { + return "multipart/form-data; boundary=" + boundary; +} + +inline std::string +serialize_multipart_formdata(const UploadFormDataItems &items, + const std::string &boundary, bool finish = true) { + std::string body; + + for (const auto &item : items) { + body += serialize_multipart_formdata_item_begin(item, boundary); + body += item.content + serialize_multipart_formdata_item_end(); + } + + if (finish) { body += serialize_multipart_formdata_finish(boundary); } + + return body; +} + +inline size_t get_multipart_content_length(const UploadFormDataItems &items, + const std::string &boundary) { + size_t total = 0; + for (const auto &item : items) { + total += serialize_multipart_formdata_item_begin(item, boundary).size(); + total += item.content.size(); + total += serialize_multipart_formdata_item_end().size(); + } + total += serialize_multipart_formdata_finish(boundary).size(); + return total; +} + +struct MultipartSegment { + const char *data; + size_t size; +}; + +// NOTE: items must outlive the returned ContentProvider +// (safe for synchronous use inside Post/Put/Patch) +inline ContentProvider +make_multipart_content_provider(const UploadFormDataItems &items, + const std::string &boundary) { + // Own the per-item header strings and the finish string + std::vector owned; + owned.reserve(items.size() + 1); + for (const auto &item : items) + owned.push_back(serialize_multipart_formdata_item_begin(item, boundary)); + owned.push_back(serialize_multipart_formdata_finish(boundary)); + + // Flat segment list: [header, content, "\r\n"] * N + [finish] + std::vector segs; + segs.reserve(items.size() * 3 + 1); + static const char crlf[] = "\r\n"; + for (size_t i = 0; i < items.size(); i++) { + segs.push_back({owned[i].data(), owned[i].size()}); + segs.push_back({items[i].content.data(), items[i].content.size()}); + segs.push_back({crlf, 2}); + } + segs.push_back({owned.back().data(), owned.back().size()}); + + struct MultipartState { + std::vector owned; + std::vector segs; + std::vector buf = std::vector(CPPHTTPLIB_SEND_BUFSIZ); + }; + auto state = std::make_shared(); + state->owned = std::move(owned); + // `segs` holds raw pointers into owned strings; std::string move preserves + // the data pointer, so these pointers remain valid after the move above. + state->segs = std::move(segs); + + return [state](size_t offset, size_t length, DataSink &sink) -> bool { + // Buffer multiple small segments into fewer, larger writes to avoid + // excessive TCP packets when there are many form data items (#2410) + auto &buf = state->buf; + auto buf_size = buf.size(); + size_t buf_len = 0; + size_t remaining = length; + + // Find the first segment containing 'offset' + size_t pos = 0; + size_t seg_idx = 0; + for (; seg_idx < state->segs.size(); seg_idx++) { + const auto &seg = state->segs[seg_idx]; + if (seg.size > 0 && offset - pos < seg.size) { break; } + pos += seg.size; + } + + size_t seg_offset = (seg_idx < state->segs.size()) ? offset - pos : 0; + + for (; seg_idx < state->segs.size() && remaining > 0; seg_idx++) { + const auto &seg = state->segs[seg_idx]; + size_t available = seg.size - seg_offset; + size_t to_copy = (std::min)(available, remaining); + const char *src = seg.data + seg_offset; + seg_offset = 0; // only the first segment has a non-zero offset + + while (to_copy > 0) { + size_t space = buf_size - buf_len; + size_t chunk = (std::min)(to_copy, space); + std::memcpy(buf.data() + buf_len, src, chunk); + buf_len += chunk; + src += chunk; + to_copy -= chunk; + remaining -= chunk; + + if (buf_len == buf_size) { + if (!sink.write(buf.data(), buf_len)) { return false; } + buf_len = 0; + } + } + } + + if (buf_len > 0) { return sink.write(buf.data(), buf_len); } + return true; + }; +} + +inline void coalesce_ranges(Ranges &ranges, size_t content_length) { + if (ranges.size() <= 1) return; + + // Sort ranges by start position + std::sort(ranges.begin(), ranges.end(), + [](const Range &a, const Range &b) { return a.first < b.first; }); + + Ranges coalesced; + coalesced.reserve(ranges.size()); + + for (auto &r : ranges) { + auto first_pos = r.first; + auto last_pos = r.second; + + // Handle special cases like in range_error + if (first_pos == -1 && last_pos == -1) { + first_pos = 0; + last_pos = static_cast(content_length); + } + + if (first_pos == -1) { + first_pos = static_cast(content_length) - last_pos; + last_pos = static_cast(content_length) - 1; + } + + if (last_pos == -1 || last_pos >= static_cast(content_length)) { + last_pos = static_cast(content_length) - 1; + } + + // Skip invalid ranges + if (!(0 <= first_pos && first_pos <= last_pos && + last_pos < static_cast(content_length))) { + continue; + } + + // Coalesce with previous range if overlapping or adjacent (but not + // identical) + if (!coalesced.empty()) { + auto &prev = coalesced.back(); + // Check if current range overlaps or is adjacent to previous range + // but don't coalesce identical ranges (allow duplicates) + if (first_pos <= prev.second + 1 && + !(first_pos == prev.first && last_pos == prev.second)) { + // Extend the previous range + prev.second = (std::max)(prev.second, last_pos); + continue; + } + } + + // Add new range + coalesced.emplace_back(first_pos, last_pos); + } + + ranges = std::move(coalesced); +} + +inline bool range_error(Request &req, Response &res) { + if (!req.ranges.empty() && 200 <= res.status && res.status < 300) { + if (res.body.empty() && res.content_provider_ && res.content_length_ == 0) { + req.ranges.clear(); + if (res.status == StatusCode::PartialContent_206) { + res.status = StatusCode::OK_200; + } + return false; + } + + ssize_t content_len = static_cast( + res.content_length_ ? res.content_length_ : res.body.size()); + + std::vector> processed_ranges; + size_t overwrapping_count = 0; + + // NOTE: The following Range check is based on '14.2. Range' in RFC 9110 + // 'HTTP Semantics' to avoid potential denial-of-service attacks. + // https://www.rfc-editor.org/rfc/rfc9110#section-14.2 + + // Too many ranges + if (req.ranges.size() > CPPHTTPLIB_RANGE_MAX_COUNT) { return true; } + + for (auto &r : req.ranges) { + auto &first_pos = r.first; + auto &last_pos = r.second; + + if (first_pos == -1 && last_pos == -1) { + first_pos = 0; + last_pos = content_len; + } + + if (first_pos == -1) { + first_pos = content_len - last_pos; + last_pos = content_len - 1; + } + + // NOTE: RFC-9110 '14.1.2. Byte Ranges': + // A client can limit the number of bytes requested without knowing the + // size of the selected representation. If the last-pos value is absent, + // or if the value is greater than or equal to the current length of the + // representation data, the byte range is interpreted as the remainder of + // the representation (i.e., the server replaces the value of last-pos + // with a value that is one less than the current length of the selected + // representation). + // https://www.rfc-editor.org/rfc/rfc9110.html#section-14.1.2-6 + if (last_pos == -1 || last_pos >= content_len) { + last_pos = content_len - 1; + } + + // Range must be within content length + if (!(0 <= first_pos && first_pos <= last_pos && + last_pos <= content_len - 1)) { + return true; + } + + // Request must not have more than two overlapping ranges + for (const auto &processed_range : processed_ranges) { + if (!(last_pos < processed_range.first || + first_pos > processed_range.second)) { + overwrapping_count++; + if (overwrapping_count > 2) { return true; } + break; // Only count once per range + } + } + + processed_ranges.emplace_back(first_pos, last_pos); + } + + // After validation, coalesce overlapping ranges as per RFC 9110 + coalesce_ranges(req.ranges, static_cast(content_len)); + } + + return false; +} + +inline std::pair +get_range_offset_and_length(Range r, size_t content_length) { + assert(r.first != -1 && r.second != -1); + assert(0 <= r.first && r.first < static_cast(content_length)); + assert(r.first <= r.second && + r.second < static_cast(content_length)); + (void)(content_length); + return std::make_pair(static_cast(r.first), + static_cast(r.second - r.first) + 1); +} + +inline std::string make_content_range_header_field( + const std::pair &offset_and_length, size_t content_length) { + auto st = offset_and_length.first; + auto ed = st + offset_and_length.second - 1; + + std::string field = "bytes "; + field += std::to_string(st); + field += '-'; + field += std::to_string(ed); + field += '/'; + field += std::to_string(content_length); + return field; +} + +template +bool process_multipart_ranges_data(const Request &req, + const std::string &boundary, + const std::string &content_type, + size_t content_length, SToken stoken, + CToken ctoken, Content content) { + for (size_t i = 0; i < req.ranges.size(); i++) { + ctoken("--"); + stoken(boundary); + ctoken("\r\n"); + if (!content_type.empty()) { + ctoken("Content-Type: "); + stoken(content_type); + ctoken("\r\n"); + } + + auto offset_and_length = + get_range_offset_and_length(req.ranges[i], content_length); + + ctoken("Content-Range: "); + stoken(make_content_range_header_field(offset_and_length, content_length)); + ctoken("\r\n"); + ctoken("\r\n"); + + if (!content(offset_and_length.first, offset_and_length.second)) { + return false; + } + ctoken("\r\n"); + } + + ctoken("--"); + stoken(boundary); + ctoken("--"); + + return true; +} + +inline void make_multipart_ranges_data(const Request &req, Response &res, + const std::string &boundary, + const std::string &content_type, + size_t content_length, + std::string &data) { + process_multipart_ranges_data( + req, boundary, content_type, content_length, + [&](const std::string &token) { data += token; }, + [&](const std::string &token) { data += token; }, + [&](size_t offset, size_t length) { + assert(offset + length <= content_length); + data += res.body.substr(offset, length); + return true; + }); +} + +inline size_t get_multipart_ranges_data_length(const Request &req, + const std::string &boundary, + const std::string &content_type, + size_t content_length) { + size_t data_length = 0; + + process_multipart_ranges_data( + req, boundary, content_type, content_length, + [&](const std::string &token) { data_length += token.size(); }, + [&](const std::string &token) { data_length += token.size(); }, + [&](size_t /*offset*/, size_t length) { + data_length += length; + return true; + }); + + return data_length; +} + +template +inline bool +write_multipart_ranges_data(Stream &strm, const Request &req, Response &res, + const std::string &boundary, + const std::string &content_type, + size_t content_length, const T &is_shutting_down) { + return process_multipart_ranges_data( + req, boundary, content_type, content_length, + [&](const std::string &token) { strm.write(token); }, + [&](const std::string &token) { strm.write(token); }, + [&](size_t offset, size_t length) { + return write_content(strm, res.content_provider_, offset, length, + is_shutting_down); + }); +} + +inline bool has_framed_body(const Request &req) { + return is_chunked_transfer_encoding(req.headers) || + req.get_header_value_u64("Content-Length") > 0; +} + +inline bool is_connection_persistent(const Request &req) { + auto conn = req.get_header_value("Connection"); + if (conn == "close") { return false; } + if (req.version == "HTTP/1.0" && conn != "Keep-Alive") { return false; } + return true; +} + +inline bool expect_content(const Request &req) { + if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH" || + req.method == "DELETE") { + return true; + } + return has_framed_body(req); +} + +#ifdef _WIN32 +class WSInit { +public: + WSInit() { + WSADATA wsaData; + if (WSAStartup(0x0002, &wsaData) == 0) is_valid_ = true; + } + + ~WSInit() { + if (is_valid_) WSACleanup(); + } + + bool is_valid_ = false; +}; + +static WSInit wsinit_; +#endif + +inline bool parse_www_authenticate(const Response &res, + std::map &auth, + bool is_proxy) { + auto auth_key = is_proxy ? "Proxy-Authenticate" : "WWW-Authenticate"; + if (res.has_header(auth_key)) { + thread_local auto re = + std::regex(R"~((?:(?:,\s*)?(.+?)=(?:"(.*?)"|([^,]*))))~"); + auto s = res.get_header_value(auth_key); + auto pos = s.find(' '); + if (pos != std::string::npos) { + auto type = s.substr(0, pos); + if (type == "Basic") { + return false; + } else if (type == "Digest") { + s = s.substr(pos + 1); + auto beg = std::sregex_iterator(s.begin(), s.end(), re); + for (auto i = beg; i != std::sregex_iterator(); ++i) { + const auto &m = *i; + auto key = s.substr(static_cast(m.position(1)), + static_cast(m.length(1))); + auto val = m.length(2) > 0 + ? s.substr(static_cast(m.position(2)), + static_cast(m.length(2))) + : s.substr(static_cast(m.position(3)), + static_cast(m.length(3))); + auth[std::move(key)] = std::move(val); + } + return true; + } + } + } + return false; +} + +class ContentProviderAdapter { +public: + explicit ContentProviderAdapter( + ContentProviderWithoutLength &&content_provider) + : content_provider_(std::move(content_provider)) {} + + bool operator()(size_t offset, size_t, DataSink &sink) { + return content_provider_(offset, sink); + } + +private: + ContentProviderWithoutLength content_provider_; +}; + +// NOTE: https://www.rfc-editor.org/rfc/rfc9110#section-5 +namespace fields { + +inline bool is_token_char(char c) { + return std::isalnum(static_cast(c)) || c == '!' || c == '#' || + c == '$' || c == '%' || c == '&' || c == '\'' || c == '*' || + c == '+' || c == '-' || c == '.' || c == '^' || c == '_' || c == '`' || + c == '|' || c == '~'; +} + +inline bool is_token(const std::string &s) { + if (s.empty()) { return false; } + for (auto c : s) { + if (!is_token_char(c)) { return false; } + } + return true; +} + +inline bool is_field_name(const std::string &s) { return is_token(s); } + +inline bool is_vchar(char c) { return c >= 33 && c <= 126; } + +inline bool is_obs_text(char c) { return 128 <= static_cast(c); } + +inline bool is_field_vchar(char c) { return is_vchar(c) || is_obs_text(c); } + +inline bool is_field_content(const std::string &s) { + if (s.empty()) { return true; } + + if (s.size() == 1) { + return is_field_vchar(s[0]); + } else if (s.size() == 2) { + return is_field_vchar(s[0]) && is_field_vchar(s[1]); + } else { + size_t i = 0; + + if (!is_field_vchar(s[i])) { return false; } + i++; + + while (i < s.size() - 1) { + auto c = s[i++]; + if (c == ' ' || c == '\t' || is_field_vchar(c)) { + } else { + return false; + } + } + + return is_field_vchar(s[i]); + } +} + +inline bool is_field_value(const std::string &s) { return is_field_content(s); } + +} // namespace fields + +inline bool perform_websocket_handshake(Stream &strm, const std::string &host, + int port, const std::string &path, + const Headers &headers, + std::string &selected_subprotocol) { + // Validate path and host + if (!fields::is_field_value(path) || !fields::is_field_value(host)) { + return false; + } + + // Validate user-provided headers + for (const auto &h : headers) { + if (!fields::is_field_name(h.first) || !fields::is_field_value(h.second)) { + return false; + } + } + + // Generate random Sec-WebSocket-Key + thread_local std::mt19937 rng(std::random_device{}()); + std::string key_bytes(16, '\0'); + for (size_t i = 0; i < 16; i += 4) { + auto r = rng(); + std::memcpy(&key_bytes[i], &r, (std::min)(size_t(4), size_t(16 - i))); + } + auto client_key = base64_encode(key_bytes); + + // Build upgrade request + std::string req_str = "GET " + path + " HTTP/1.1\r\n"; + req_str += "Host: " + host + ":" + std::to_string(port) + "\r\n"; + req_str += "Upgrade: websocket\r\n"; + req_str += "Connection: Upgrade\r\n"; + req_str += "Sec-WebSocket-Key: " + client_key + "\r\n"; + req_str += "Sec-WebSocket-Version: 13\r\n"; + for (const auto &h : headers) { + req_str += h.first + ": " + h.second + "\r\n"; + } + req_str += "\r\n"; + + if (strm.write(req_str.data(), req_str.size()) < 0) { return false; } + + // Verify 101 response and Sec-WebSocket-Accept header + auto expected_accept = websocket_accept_key(client_key); + return read_websocket_upgrade_response(strm, expected_accept, + selected_subprotocol); +} + +} // namespace detail + +/* + * Group 2: detail namespace - SSL common utilities + */ + +#ifdef CPPHTTPLIB_SSL_ENABLED +namespace detail { + +class SSLSocketStream final : public Stream { +public: + SSLSocketStream( + socket_t sock, tls::session_t session, time_t read_timeout_sec, + time_t read_timeout_usec, time_t write_timeout_sec, + time_t write_timeout_usec, time_t max_timeout_msec = 0, + std::chrono::time_point start_time = + (std::chrono::steady_clock::time_point::min)()); + ~SSLSocketStream() override; + + bool is_readable() const override; + bool wait_readable() const override; + bool wait_writable() const override; + bool is_peer_alive() const override; + ssize_t read(char *ptr, size_t size) override; + ssize_t write(const char *ptr, size_t size) override; + void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; + socket_t socket() const override; + time_t duration() const override; + void set_read_timeout(time_t sec, time_t usec = 0) override; + +private: + socket_t sock_; + tls::session_t session_; + time_t read_timeout_sec_; + time_t read_timeout_usec_; + time_t write_timeout_sec_; + time_t write_timeout_usec_; + time_t max_timeout_msec_; + const std::chrono::time_point start_time_; +}; + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline std::string message_digest(const std::string &s, const EVP_MD *algo) { + auto context = std::unique_ptr( + EVP_MD_CTX_new(), EVP_MD_CTX_free); + + unsigned int hash_length = 0; + unsigned char hash[EVP_MAX_MD_SIZE]; + + EVP_DigestInit_ex(context.get(), algo, nullptr); + EVP_DigestUpdate(context.get(), s.c_str(), s.size()); + EVP_DigestFinal_ex(context.get(), hash, &hash_length); + + std::stringstream ss; + for (auto i = 0u; i < hash_length; ++i) { + ss << std::hex << std::setw(2) << std::setfill('0') + << static_cast(hash[i]); + } + + return ss.str(); +} + +inline std::string MD5(const std::string &s) { + return message_digest(s, EVP_md5()); +} + +inline std::string SHA_256(const std::string &s) { + return message_digest(s, EVP_sha256()); +} + +inline std::string SHA_512(const std::string &s) { + return message_digest(s, EVP_sha512()); +} +#elif defined(CPPHTTPLIB_MBEDTLS_SUPPORT) +namespace { +template +inline std::string hash_to_hex(const unsigned char (&hash)[N]) { + std::stringstream ss; + for (size_t i = 0; i < N; ++i) { + ss << std::hex << std::setw(2) << std::setfill('0') + << static_cast(hash[i]); + } + return ss.str(); +} +} // namespace + +inline std::string MD5(const std::string &s) { + unsigned char hash[16]; +#ifdef CPPHTTPLIB_MBEDTLS_V3 + mbedtls_md5(reinterpret_cast(s.c_str()), s.size(), + hash); +#else + mbedtls_md5_ret(reinterpret_cast(s.c_str()), s.size(), + hash); +#endif + return hash_to_hex(hash); +} + +inline std::string SHA_256(const std::string &s) { + unsigned char hash[32]; +#ifdef CPPHTTPLIB_MBEDTLS_V3 + mbedtls_sha256(reinterpret_cast(s.c_str()), s.size(), + hash, 0); +#else + mbedtls_sha256_ret(reinterpret_cast(s.c_str()), + s.size(), hash, 0); +#endif + return hash_to_hex(hash); +} + +inline std::string SHA_512(const std::string &s) { + unsigned char hash[64]; +#ifdef CPPHTTPLIB_MBEDTLS_V3 + mbedtls_sha512(reinterpret_cast(s.c_str()), s.size(), + hash, 0); +#else + mbedtls_sha512_ret(reinterpret_cast(s.c_str()), + s.size(), hash, 0); +#endif + return hash_to_hex(hash); +} +#elif defined(CPPHTTPLIB_WOLFSSL_SUPPORT) +namespace { +template +inline std::string hash_to_hex(const unsigned char (&hash)[N]) { + std::stringstream ss; + for (size_t i = 0; i < N; ++i) { + ss << std::hex << std::setw(2) << std::setfill('0') + << static_cast(hash[i]); + } + return ss.str(); +} +} // namespace + +inline std::string MD5(const std::string &s) { + unsigned char hash[WC_MD5_DIGEST_SIZE]; + wc_Md5Hash(reinterpret_cast(s.c_str()), + static_cast(s.size()), hash); + return hash_to_hex(hash); +} + +inline std::string SHA_256(const std::string &s) { + unsigned char hash[WC_SHA256_DIGEST_SIZE]; + wc_Sha256Hash(reinterpret_cast(s.c_str()), + static_cast(s.size()), hash); + return hash_to_hex(hash); +} + +inline std::string SHA_512(const std::string &s) { + unsigned char hash[WC_SHA512_DIGEST_SIZE]; + wc_Sha512Hash(reinterpret_cast(s.c_str()), + static_cast(s.size()), hash); + return hash_to_hex(hash); +} +#endif + +inline bool is_ip_address(const std::string &host) { + struct in_addr addr4; + struct in6_addr addr6; + return inet_pton(AF_INET, host.c_str(), &addr4) == 1 || + inet_pton(AF_INET6, host.c_str(), &addr6) == 1; +} + +template +inline bool process_server_socket_ssl( + const std::atomic &svr_sock, tls::session_t session, + socket_t sock, size_t keep_alive_max_count, time_t keep_alive_timeout_sec, + time_t read_timeout_sec, time_t read_timeout_usec, time_t write_timeout_sec, + time_t write_timeout_usec, T callback) { + return process_server_socket_core( + svr_sock, sock, keep_alive_max_count, keep_alive_timeout_sec, + [&](bool close_connection, bool &connection_closed) { + SSLSocketStream strm(sock, session, read_timeout_sec, read_timeout_usec, + write_timeout_sec, write_timeout_usec); + return callback(strm, close_connection, connection_closed); + }); +} + +template +inline bool process_client_socket_ssl( + tls::session_t session, socket_t sock, time_t read_timeout_sec, + time_t read_timeout_usec, time_t write_timeout_sec, + time_t write_timeout_usec, time_t max_timeout_msec, + std::chrono::time_point start_time, T callback) { + SSLSocketStream strm(sock, session, read_timeout_sec, read_timeout_usec, + write_timeout_sec, write_timeout_usec, max_timeout_msec, + start_time); + return callback(strm); +} + +inline std::pair make_digest_authentication_header( + const Request &req, const std::map &auth, + size_t cnonce_count, const std::string &cnonce, const std::string &username, + const std::string &password, bool is_proxy = false) { + std::string nc; + { + std::stringstream ss; + ss << std::setfill('0') << std::setw(8) << std::hex << cnonce_count; + nc = ss.str(); + } + + std::string qop; + if (auth.find("qop") != auth.end()) { + qop = auth.at("qop"); + if (qop.find("auth-int") != std::string::npos) { + qop = "auth-int"; + } else if (qop.find("auth") != std::string::npos) { + qop = "auth"; + } else { + qop.clear(); + } + } + + std::string algo = "MD5"; + if (auth.find("algorithm") != auth.end()) { algo = auth.at("algorithm"); } + + std::string response; + { + auto H = algo == "SHA-256" ? detail::SHA_256 + : algo == "SHA-512" ? detail::SHA_512 + : detail::MD5; + + auto A1 = username + ":" + auth.at("realm") + ":" + password; + + auto A2 = req.method + ":" + req.path; + if (qop == "auth-int") { A2 += ":" + H(req.body); } + + if (qop.empty()) { + response = H(H(A1) + ":" + auth.at("nonce") + ":" + H(A2)); + } else { + response = H(H(A1) + ":" + auth.at("nonce") + ":" + nc + ":" + cnonce + + ":" + qop + ":" + H(A2)); + } + } + + auto opaque = (auth.find("opaque") != auth.end()) ? auth.at("opaque") : ""; + + auto field = "Digest username=\"" + username + "\", realm=\"" + + auth.at("realm") + "\", nonce=\"" + auth.at("nonce") + + "\", uri=\"" + req.path + "\", algorithm=" + algo + + (qop.empty() ? ", response=\"" + : ", qop=" + qop + ", nc=" + nc + ", cnonce=\"" + + cnonce + "\", response=\"") + + response + "\"" + + (opaque.empty() ? "" : ", opaque=\"" + opaque + "\""); + + auto key = is_proxy ? "Proxy-Authorization" : "Authorization"; + return std::make_pair(key, field); +} + +inline bool match_hostname(const std::string &pattern, + const std::string &hostname) { + // Exact match (case-insensitive) + if (detail::case_ignore::equal(hostname, pattern)) { return true; } + + // Split both pattern and hostname into components by '.' + std::vector pattern_components; + if (!pattern.empty()) { + split(pattern.data(), pattern.data() + pattern.size(), '.', + [&](const char *b, const char *e) { + pattern_components.emplace_back(b, e); + }); + } + + std::vector host_components; + if (!hostname.empty()) { + split(hostname.data(), hostname.data() + hostname.size(), '.', + [&](const char *b, const char *e) { + host_components.emplace_back(b, e); + }); + } + + // Component count must match + if (host_components.size() != pattern_components.size()) { return false; } + + // Compare each component with wildcard support + // Supports: "*" (full wildcard), "prefix*" (partial wildcard) + // https://bugs.launchpad.net/ubuntu/+source/firefox-3.0/+bug/376484 + auto itr = pattern_components.begin(); + for (const auto &h : host_components) { + auto &p = *itr; + if (!detail::case_ignore::equal(p, h) && p != "*") { + bool partial_match = false; + if (!p.empty() && p[p.size() - 1] == '*') { + const auto prefix_length = p.size() - 1; + if (prefix_length == 0) { + partial_match = true; + } else if (h.size() >= prefix_length) { + partial_match = + std::equal(p.begin(), + p.begin() + static_cast( + prefix_length), + h.begin(), [](const char ca, const char cb) { + return detail::case_ignore::to_lower(ca) == + detail::case_ignore::to_lower(cb); + }); + } + } + if (!partial_match) { return false; } + } + ++itr; + } + + return true; +} + +#ifdef _WIN32 +// Verify certificate using Windows CertGetCertificateChain API. +// This provides real-time certificate validation with Windows Update +// integration, independent of the TLS backend (OpenSSL or MbedTLS). +inline bool +verify_cert_with_windows_schannel(const std::vector &der_cert, + const std::string &hostname, + bool verify_hostname, uint64_t &out_error) { + if (der_cert.empty()) { return false; } + + out_error = 0; + + // Create Windows certificate context from DER data + auto cert_context = CertCreateCertificateContext( + X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, der_cert.data(), + static_cast(der_cert.size())); + + if (!cert_context) { + out_error = GetLastError(); + return false; + } + + auto cert_guard = + scope_exit([&] { CertFreeCertificateContext(cert_context); }); + + // Setup chain parameters + CERT_CHAIN_PARA chain_para = {}; + chain_para.cbSize = sizeof(chain_para); + + // Build certificate chain with revocation checking + PCCERT_CHAIN_CONTEXT chain_context = nullptr; + auto chain_result = CertGetCertificateChain( + nullptr, cert_context, nullptr, cert_context->hCertStore, &chain_para, + CERT_CHAIN_CACHE_END_CERT | CERT_CHAIN_REVOCATION_CHECK_END_CERT | + CERT_CHAIN_REVOCATION_ACCUMULATIVE_TIMEOUT, + nullptr, &chain_context); + + if (!chain_result || !chain_context) { + out_error = GetLastError(); + return false; + } + + auto chain_guard = + scope_exit([&] { CertFreeCertificateChain(chain_context); }); + + // Check if chain has errors + if (chain_context->TrustStatus.dwErrorStatus != CERT_TRUST_NO_ERROR) { + out_error = chain_context->TrustStatus.dwErrorStatus; + return false; + } + + // Verify SSL policy + SSL_EXTRA_CERT_CHAIN_POLICY_PARA extra_policy_para = {}; + extra_policy_para.cbSize = sizeof(extra_policy_para); +#ifdef AUTHTYPE_SERVER + extra_policy_para.dwAuthType = AUTHTYPE_SERVER; +#endif + + std::wstring whost; + if (verify_hostname) { + whost = u8string_to_wstring(hostname.c_str()); + extra_policy_para.pwszServerName = const_cast(whost.c_str()); + } + + CERT_CHAIN_POLICY_PARA policy_para = {}; + policy_para.cbSize = sizeof(policy_para); +#ifdef CERT_CHAIN_POLICY_IGNORE_ALL_REV_UNKNOWN_FLAGS + policy_para.dwFlags = CERT_CHAIN_POLICY_IGNORE_ALL_REV_UNKNOWN_FLAGS; +#else + policy_para.dwFlags = 0; +#endif + policy_para.pvExtraPolicyPara = &extra_policy_para; + + CERT_CHAIN_POLICY_STATUS policy_status = {}; + policy_status.cbSize = sizeof(policy_status); + + if (!CertVerifyCertificateChainPolicy(CERT_CHAIN_POLICY_SSL, chain_context, + &policy_para, &policy_status)) { + out_error = GetLastError(); + return false; + } + + if (policy_status.dwError != 0) { + out_error = policy_status.dwError; + return false; + } + + return true; +} +#endif // _WIN32 + +// Loads CA file/dir configuration and applies the system CA policy to a +// client TLS context. PEM data and native stores are applied to the context +// directly at set time; has_custom_store reflects them for the Auto policy +// decision. +inline bool load_client_ca_config(tls::ctx_t ctx, + const std::string &ca_cert_file_path, + const std::string &ca_cert_dir_path, + bool has_custom_store, SystemCAMode mode, + uint64_t &backend_error) { + auto ret = true; + + if (!ca_cert_file_path.empty()) { + if (!tls::load_ca_file(ctx, ca_cert_file_path.c_str())) { + backend_error = tls::get_error(); + ret = false; + } + } else if (!ca_cert_dir_path.empty()) { + if (!tls::load_ca_dir(ctx, ca_cert_dir_path.c_str())) { + backend_error = tls::get_error(); + ret = false; + } + } + + auto has_custom_ca = !ca_cert_file_path.empty() || + !ca_cert_dir_path.empty() || has_custom_store; + if (mode == SystemCAMode::Enabled || + (mode == SystemCAMode::Auto && !has_custom_ca)) { + if (!tls::load_system_certs(ctx)) { backend_error = tls::get_error(); } + } + + return ret; +} + +inline bool setup_client_tls_session(const std::string &host, tls::ctx_t ctx, + tls::session_t &session, socket_t sock, + bool server_certificate_verification, + time_t timeout_sec, time_t timeout_usec) { + using namespace tls; + + if (!ctx) { return false; } + + bool is_ip = is_ip_address(host); + +#if defined(CPPHTTPLIB_MBEDTLS_SUPPORT) || defined(CPPHTTPLIB_WOLFSSL_SUPPORT) + // Chain verification happens during the handshake even for IP hosts; the + // certificate identity is verified post-handshake via verify_hostname() + set_verify_client(ctx, server_certificate_verification); +#endif + + session = create_session(ctx, sock); + if (!session) { return false; } + + // RFC 6066: SNI must not be set for IP addresses. On Mbed TLS and wolfSSL + // set_hostname also sets SNI, so it must be skipped for IP hosts as well; + // their identity is checked post-handshake below instead. + if (!is_ip) { + if (server_certificate_verification) { + set_hostname(session, host.c_str()); + } else { + set_sni(session, host.c_str()); + } + } + + if (!connect_nonblocking(session, sock, timeout_sec, timeout_usec, nullptr)) { + return false; + } + + if (server_certificate_verification) { + if (get_verify_result(session) != 0) { return false; } + + // Identity check against the peer certificate, post-handshake for all + // backends (same as SSLClient). For IP hosts this is the only identity + // verification since no hostname is bound during the handshake. + auto server_cert = get_peer_cert(session); + if (!server_cert) { return false; } + auto cert_guard = detail::scope_exit([&] { free_cert(server_cert); }); + if (!verify_hostname(server_cert, host.c_str())) { return false; } + } + + return true; +} + +} // namespace detail +#endif // CPPHTTPLIB_SSL_ENABLED + +/* + * Group 3: httplib namespace - Non-SSL public API implementations + */ + +inline void default_socket_options(socket_t sock) { + set_socket_opt(sock, SOL_SOCKET, +#ifdef SO_REUSEPORT + SO_REUSEPORT, +#else + SO_REUSEADDR, +#endif + 1); +} + +inline bool set_socket_opt(socket_t sock, int level, int optname, int optval) { + return detail::set_socket_opt_impl(sock, level, optname, &optval, + sizeof(optval)); +} + +inline std::string get_bearer_token_auth(const Request &req) { + if (req.has_header("Authorization")) { + constexpr auto bearer_header_prefix_len = detail::str_len("Bearer "); + return req.get_header_value("Authorization") + .substr(bearer_header_prefix_len); + } + return ""; +} + +inline const char *status_message(int status) { + switch (status) { + case StatusCode::Continue_100: return "Continue"; + case StatusCode::SwitchingProtocol_101: return "Switching Protocol"; + case StatusCode::Processing_102: return "Processing"; + case StatusCode::EarlyHints_103: return "Early Hints"; + case StatusCode::OK_200: return "OK"; + case StatusCode::Created_201: return "Created"; + case StatusCode::Accepted_202: return "Accepted"; + case StatusCode::NonAuthoritativeInformation_203: + return "Non-Authoritative Information"; + case StatusCode::NoContent_204: return "No Content"; + case StatusCode::ResetContent_205: return "Reset Content"; + case StatusCode::PartialContent_206: return "Partial Content"; + case StatusCode::MultiStatus_207: return "Multi-Status"; + case StatusCode::AlreadyReported_208: return "Already Reported"; + case StatusCode::IMUsed_226: return "IM Used"; + case StatusCode::MultipleChoices_300: return "Multiple Choices"; + case StatusCode::MovedPermanently_301: return "Moved Permanently"; + case StatusCode::Found_302: return "Found"; + case StatusCode::SeeOther_303: return "See Other"; + case StatusCode::NotModified_304: return "Not Modified"; + case StatusCode::UseProxy_305: return "Use Proxy"; + case StatusCode::unused_306: return "unused"; + case StatusCode::TemporaryRedirect_307: return "Temporary Redirect"; + case StatusCode::PermanentRedirect_308: return "Permanent Redirect"; + case StatusCode::BadRequest_400: return "Bad Request"; + case StatusCode::Unauthorized_401: return "Unauthorized"; + case StatusCode::PaymentRequired_402: return "Payment Required"; + case StatusCode::Forbidden_403: return "Forbidden"; + case StatusCode::NotFound_404: return "Not Found"; + case StatusCode::MethodNotAllowed_405: return "Method Not Allowed"; + case StatusCode::NotAcceptable_406: return "Not Acceptable"; + case StatusCode::ProxyAuthenticationRequired_407: + return "Proxy Authentication Required"; + case StatusCode::RequestTimeout_408: return "Request Timeout"; + case StatusCode::Conflict_409: return "Conflict"; + case StatusCode::Gone_410: return "Gone"; + case StatusCode::LengthRequired_411: return "Length Required"; + case StatusCode::PreconditionFailed_412: return "Precondition Failed"; + case StatusCode::PayloadTooLarge_413: return "Payload Too Large"; + case StatusCode::UriTooLong_414: return "URI Too Long"; + case StatusCode::UnsupportedMediaType_415: return "Unsupported Media Type"; + case StatusCode::RangeNotSatisfiable_416: return "Range Not Satisfiable"; + case StatusCode::ExpectationFailed_417: return "Expectation Failed"; + case StatusCode::ImATeapot_418: return "I'm a teapot"; + case StatusCode::MisdirectedRequest_421: return "Misdirected Request"; + case StatusCode::UnprocessableContent_422: return "Unprocessable Content"; + case StatusCode::Locked_423: return "Locked"; + case StatusCode::FailedDependency_424: return "Failed Dependency"; + case StatusCode::TooEarly_425: return "Too Early"; + case StatusCode::UpgradeRequired_426: return "Upgrade Required"; + case StatusCode::PreconditionRequired_428: return "Precondition Required"; + case StatusCode::TooManyRequests_429: return "Too Many Requests"; + case StatusCode::RequestHeaderFieldsTooLarge_431: + return "Request Header Fields Too Large"; + case StatusCode::UnavailableForLegalReasons_451: + return "Unavailable For Legal Reasons"; + case StatusCode::NotImplemented_501: return "Not Implemented"; + case StatusCode::BadGateway_502: return "Bad Gateway"; + case StatusCode::ServiceUnavailable_503: return "Service Unavailable"; + case StatusCode::GatewayTimeout_504: return "Gateway Timeout"; + case StatusCode::HttpVersionNotSupported_505: + return "HTTP Version Not Supported"; + case StatusCode::VariantAlsoNegotiates_506: return "Variant Also Negotiates"; + case StatusCode::InsufficientStorage_507: return "Insufficient Storage"; + case StatusCode::LoopDetected_508: return "Loop Detected"; + case StatusCode::NotExtended_510: return "Not Extended"; + case StatusCode::NetworkAuthenticationRequired_511: + return "Network Authentication Required"; + + default: + case StatusCode::InternalServerError_500: return "Internal Server Error"; + } +} + +inline std::string to_string(const Error error) { + switch (error) { + case Error::Success: return "Success (no error)"; + case Error::Unknown: return "Unknown"; + case Error::Connection: return "Could not establish connection"; + case Error::BindIPAddress: return "Failed to bind IP address"; + case Error::Read: return "Failed to read connection"; + case Error::Write: return "Failed to write connection"; + case Error::ExceedRedirectCount: return "Maximum redirect count exceeded"; + case Error::Canceled: return "Connection handling canceled"; + case Error::SSLConnection: return "SSL connection failed"; + case Error::SSLLoadingCerts: return "SSL certificate loading failed"; + case Error::SSLServerVerification: return "SSL server verification failed"; + case Error::SSLServerHostnameVerification: + return "SSL server hostname verification failed"; + case Error::UnsupportedMultipartBoundaryChars: + return "Unsupported HTTP multipart boundary characters"; + case Error::Compression: return "Compression failed"; + case Error::ConnectionTimeout: return "Connection timed out"; + case Error::ProxyConnection: return "Proxy connection failed"; + case Error::ConnectionClosed: return "Connection closed by server"; + case Error::Timeout: return "Read timeout"; + case Error::ResourceExhaustion: return "Resource exhaustion"; + case Error::TooManyFormDataFiles: return "Too many form data files"; + case Error::ExceedMaxPayloadSize: return "Exceeded maximum payload size"; + case Error::ExceedUriMaxLength: return "Exceeded maximum URI length"; + case Error::ExceedMaxSocketDescriptorCount: + return "Exceeded maximum socket descriptor count"; + case Error::InvalidRequestLine: return "Invalid request line"; + case Error::InvalidHTTPMethod: return "Invalid HTTP method"; + case Error::InvalidHTTPVersion: return "Invalid HTTP version"; + case Error::InvalidHeaders: return "Invalid headers"; + case Error::MultipartParsing: return "Multipart parsing failed"; + case Error::OpenFile: return "Failed to open file"; + case Error::Listen: return "Failed to listen on socket"; + case Error::GetSockName: return "Failed to get socket name"; + case Error::UnsupportedAddressFamily: return "Unsupported address family"; + case Error::HTTPParsing: return "HTTP parsing failed"; + case Error::InvalidRangeHeader: return "Invalid Range header"; + default: break; + } + + return "Invalid"; +} + +inline std::ostream &operator<<(std::ostream &os, const Error &obj) { + os << to_string(obj); + os << " (" << static_cast::type>(obj) << ')'; + return os; +} + +inline std::string hosted_at(const std::string &hostname) { + std::vector addrs; + hosted_at(hostname, addrs); + if (addrs.empty()) { return std::string(); } + return addrs[0]; +} + +inline void hosted_at(const std::string &hostname, + std::vector &addrs) { + struct addrinfo hints; + struct addrinfo *result; + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = 0; + + if (detail::getaddrinfo_with_timeout(hostname.c_str(), nullptr, &hints, + &result, 0)) { +#if defined __linux__ && !defined __ANDROID__ + res_init(); +#endif + return; + } + auto se = detail::scope_exit([&] { freeaddrinfo(result); }); + + for (auto rp = result; rp; rp = rp->ai_next) { + const auto &addr = + *reinterpret_cast(rp->ai_addr); + std::string ip; + auto dummy = -1; + if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip, + dummy)) { + addrs.emplace_back(std::move(ip)); + } + } +} + +inline std::string encode_uri_component(const std::string &value) { + std::ostringstream escaped; + escaped.fill('0'); + escaped << std::hex; + + for (auto c : value) { + if (std::isalnum(static_cast(c)) || c == '-' || c == '_' || + c == '.' || c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' || + c == ')') { + escaped << c; + } else { + escaped << std::uppercase; + escaped << '%' << std::setw(2) + << static_cast(static_cast(c)); + escaped << std::nouppercase; + } + } + + return escaped.str(); +} + +inline std::string encode_uri(const std::string &value) { + std::ostringstream escaped; + escaped.fill('0'); + escaped << std::hex; + + for (auto c : value) { + if (std::isalnum(static_cast(c)) || c == '-' || c == '_' || + c == '.' || c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' || + c == ')' || c == ';' || c == '/' || c == '?' || c == ':' || c == '@' || + c == '&' || c == '=' || c == '+' || c == '$' || c == ',' || c == '#') { + escaped << c; + } else { + escaped << std::uppercase; + escaped << '%' << std::setw(2) + << static_cast(static_cast(c)); + escaped << std::nouppercase; + } + } + + return escaped.str(); +} + +inline std::string decode_uri_component(const std::string &value) { + std::string result; + + for (size_t i = 0; i < value.size(); i++) { + if (value[i] == '%' && i + 2 < value.size()) { + auto val = 0; + if (detail::from_hex_to_i(value, i + 1, 2, val)) { + result += static_cast(val); + i += 2; + } else { + result += value[i]; + } + } else { + result += value[i]; + } + } + + return result; +} + +inline std::string decode_uri(const std::string &value) { + std::string result; + + for (size_t i = 0; i < value.size(); i++) { + if (value[i] == '%' && i + 2 < value.size()) { + auto val = 0; + if (detail::from_hex_to_i(value, i + 1, 2, val)) { + result += static_cast(val); + i += 2; + } else { + result += value[i]; + } + } else { + result += value[i]; + } + } + + return result; +} + +inline std::string encode_path_component(const std::string &component) { + std::string result; + result.reserve(component.size() * 3); + + for (size_t i = 0; i < component.size(); i++) { + auto c = static_cast(component[i]); + + // Unreserved characters per RFC 3986: ALPHA / DIGIT / "-" / "." / "_" / "~" + if (std::isalnum(c) || c == '-' || c == '.' || c == '_' || c == '~') { + result += static_cast(c); + } + // Path-safe sub-delimiters: "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / + // "," / ";" / "=" + else if (c == '!' || c == '$' || c == '&' || c == '\'' || c == '(' || + c == ')' || c == '*' || c == '+' || c == ',' || c == ';' || + c == '=') { + result += static_cast(c); + } + // Colon is allowed in path segments except first segment + else if (c == ':') { + result += static_cast(c); + } + // @ is allowed in path + else if (c == '@') { + result += static_cast(c); + } else { + result += '%'; + char hex[3]; + snprintf(hex, sizeof(hex), "%02X", c); + result.append(hex, 2); + } + } + return result; +} + +inline std::string decode_path_component(const std::string &component) { + std::string result; + result.reserve(component.size()); + + for (size_t i = 0; i < component.size(); i++) { + if (component[i] == '%' && i + 1 < component.size()) { + if (component[i + 1] == 'u') { + // Unicode %uXXXX encoding + auto val = 0; + if (detail::from_hex_to_i(component, i + 2, 4, val)) { + // 4 digits Unicode codes: val is 0x0000-0xFFFF (from 4 hex digits), + // so to_utf8 writes at most 3 bytes. buff[4] is safe. + char buff[4]; + size_t len = detail::to_utf8(val, buff); + if (len > 0) { result.append(buff, len); } + i += 5; // 'u0000' + } else { + result += component[i]; + } + } else { + // Standard %XX encoding + auto val = 0; + if (detail::from_hex_to_i(component, i + 1, 2, val)) { + // 2 digits hex codes + result += static_cast(val); + i += 2; // 'XX' + } else { + result += component[i]; + } + } + } else { + result += component[i]; + } + } + return result; +} + +inline std::string encode_query_component(const std::string &component, + bool space_as_plus) { + std::string result; + result.reserve(component.size() * 3); + + for (size_t i = 0; i < component.size(); i++) { + auto c = static_cast(component[i]); + + // Unreserved characters per RFC 3986 + if (std::isalnum(c) || c == '-' || c == '.' || c == '_' || c == '~') { + result += static_cast(c); + } + // Space handling + else if (c == ' ') { + if (space_as_plus) { + result += '+'; + } else { + result += "%20"; + } + } + // Plus sign handling + else if (c == '+') { + if (space_as_plus) { + result += "%2B"; + } else { + result += static_cast(c); + } + } + // Query-safe sub-delimiters (excluding & and = which are query delimiters) + else if (c == '!' || c == '$' || c == '\'' || c == '(' || c == ')' || + c == '*' || c == ',' || c == ';') { + result += static_cast(c); + } + // Colon and @ are allowed in query + else if (c == ':' || c == '@') { + result += static_cast(c); + } + // Forward slash is allowed in query values + else if (c == '/') { + result += static_cast(c); + } + // Question mark is allowed in query values (after first ?) + else if (c == '?') { + result += static_cast(c); + } else { + result += '%'; + char hex[3]; + snprintf(hex, sizeof(hex), "%02X", c); + result.append(hex, 2); + } + } + return result; +} + +inline std::string decode_query_component(const std::string &component, + bool plus_as_space) { + std::string result; + result.reserve(component.size()); + + for (size_t i = 0; i < component.size(); i++) { + if (component[i] == '%' && i + 2 < component.size()) { + auto val = 0; + if (detail::from_hex_to_i(component, i + 1, 2, val)) { + result += static_cast(val); + i += 2; + } else { + result += component[i]; + } + } else if (component[i] == '+' && plus_as_space) { + result += ' '; // + becomes space in form-urlencoded + } else { + result += component[i]; + } + } + return result; +} + +inline std::string sanitize_filename(const std::string &filename) { + // Extract basename: find the last path separator (/ or \) + auto pos = filename.find_last_of("/\\"); + auto result = + (pos != std::string::npos) ? filename.substr(pos + 1) : filename; + + // Strip null bytes + result.erase(std::remove(result.begin(), result.end(), '\0'), result.end()); + + // Trim whitespace + { + auto start = result.find_first_not_of(" \t"); + auto end = result.find_last_not_of(" \t"); + result = (start == std::string::npos) + ? "" + : result.substr(start, end - start + 1); + } + + // Reject . and .. + if (result == "." || result == "..") { return ""; } + + return result; +} + +inline std::string append_query_params(const std::string &path, + const Params ¶ms) { + std::string path_with_query = path; + thread_local const std::regex re("[^?]+\\?.*"); + auto delm = std::regex_match(path, re) ? '&' : '?'; + path_with_query += delm + detail::params_to_query_str(params); + return path_with_query; +} + +// Header utilities +inline std::pair +make_range_header(const Ranges &ranges) { + std::string field = "bytes="; + auto i = 0; + for (const auto &r : ranges) { + if (i != 0) { field += ", "; } + if (r.first != -1) { field += std::to_string(r.first); } + field += '-'; + if (r.second != -1) { field += std::to_string(r.second); } + i++; + } + return std::make_pair("Range", std::move(field)); +} + +inline std::pair +make_basic_authentication_header(const std::string &username, + const std::string &password, bool is_proxy) { + auto field = "Basic " + detail::base64_encode(username + ":" + password); + auto key = is_proxy ? "Proxy-Authorization" : "Authorization"; + return std::make_pair(key, std::move(field)); +} + +inline std::pair +make_bearer_token_authentication_header(const std::string &token, + bool is_proxy = false) { + auto field = "Bearer " + token; + auto key = is_proxy ? "Proxy-Authorization" : "Authorization"; + return std::make_pair(key, std::move(field)); +} + +// Request implementation +inline size_t Request::get_header_value_u64(const std::string &key, size_t def, + size_t id) const { + return detail::get_header_value_u64(headers, key, def, id); +} + +inline bool Request::has_header(const std::string &key) const { + return detail::has_header(headers, key); +} + +inline std::string Request::get_header_value(const std::string &key, + const char *def, size_t id) const { + return detail::get_header_value(headers, key, def, id); +} + +inline size_t Request::get_header_value_count(const std::string &key) const { + return detail::get_header_value_count(headers, key); +} + +inline void Request::set_header(const std::string &key, + const std::string &val) { + detail::set_header(headers, key, val); +} + +inline bool Request::has_trailer(const std::string &key) const { + return trailers.find(key) != trailers.end(); +} + +inline std::string Request::get_trailer_value(const std::string &key, + size_t id) const { + return detail::get_multimap_value(trailers, key, id); +} + +inline size_t Request::get_trailer_value_count(const std::string &key) const { + auto r = trailers.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +inline bool Request::has_param(const std::string &key) const { + return params.find(key) != params.end(); +} + +inline std::string Request::get_param_value(const std::string &key, + size_t id) const { + return detail::get_multimap_value(params, key, id); +} + +inline std::vector +Request::get_param_values(const std::string &key) const { + auto rng = params.equal_range(key); + std::vector values; + values.reserve(static_cast(std::distance(rng.first, rng.second))); + for (auto it = rng.first; it != rng.second; ++it) { + values.push_back(it->second); + } + return values; +} + +inline size_t Request::get_param_value_count(const std::string &key) const { + auto r = params.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +inline bool Request::is_multipart_form_data() const { + const auto &content_type = get_header_value("Content-Type"); + return detail::extract_media_type(content_type) == "multipart/form-data"; +} + +// Multipart FormData implementation +inline std::string MultipartFormData::get_field(const std::string &key, + size_t id) const { + auto rng = fields.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { return it->second.content; } + return std::string(); +} + +inline std::vector +MultipartFormData::get_fields(const std::string &key) const { + std::vector values; + auto rng = fields.equal_range(key); + for (auto it = rng.first; it != rng.second; it++) { + values.push_back(it->second.content); + } + return values; +} + +inline bool MultipartFormData::has_field(const std::string &key) const { + return fields.find(key) != fields.end(); +} + +inline size_t MultipartFormData::get_field_count(const std::string &key) const { + auto r = fields.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +inline FormData MultipartFormData::get_file(const std::string &key, + size_t id) const { + return detail::get_multimap_value(files, key, id); +} + +inline std::vector +MultipartFormData::get_files(const std::string &key) const { + std::vector values; + auto rng = files.equal_range(key); + for (auto it = rng.first; it != rng.second; it++) { + values.push_back(it->second); + } + return values; +} + +inline bool MultipartFormData::has_file(const std::string &key) const { + return files.find(key) != files.end(); +} + +inline size_t MultipartFormData::get_file_count(const std::string &key) const { + auto r = files.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +// Response implementation +inline size_t Response::get_header_value_u64(const std::string &key, size_t def, + size_t id) const { + return detail::get_header_value_u64(headers, key, def, id); +} + +inline bool Response::has_header(const std::string &key) const { + return headers.find(key) != headers.end(); +} + +inline std::string Response::get_header_value(const std::string &key, + const char *def, + size_t id) const { + return detail::get_header_value(headers, key, def, id); +} + +inline size_t Response::get_header_value_count(const std::string &key) const { + return detail::get_header_value_count(headers, key); +} + +inline void Response::set_header(const std::string &key, + const std::string &val) { + detail::set_header(headers, key, val); +} +inline bool Response::has_trailer(const std::string &key) const { + return trailers.find(key) != trailers.end(); +} + +inline std::string Response::get_trailer_value(const std::string &key, + size_t id) const { + return detail::get_multimap_value(trailers, key, id); +} + +inline size_t Response::get_trailer_value_count(const std::string &key) const { + auto r = trailers.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +inline void Response::set_redirect(const std::string &url, int stat) { + if (detail::fields::is_field_value(url)) { + set_header("Location", url); + if (300 <= stat && stat < 400) { + this->status = stat; + } else { + this->status = StatusCode::Found_302; + } + } +} + +inline void Response::set_content(const char *s, size_t n, + const std::string &content_type) { + body.assign(s, n); + + auto rng = headers.equal_range("Content-Type"); + headers.erase(rng.first, rng.second); + set_header("Content-Type", content_type); +} + +inline void Response::set_content(const std::string &s, + const std::string &content_type) { + set_content(s.data(), s.size(), content_type); +} + +inline void Response::set_content(std::string &&s, + const std::string &content_type) { + body = std::move(s); + + auto rng = headers.equal_range("Content-Type"); + headers.erase(rng.first, rng.second); + set_header("Content-Type", content_type); +} + +inline void Response::set_content_provider( + size_t in_length, const std::string &content_type, ContentProvider provider, + ContentProviderResourceReleaser resource_releaser) { + set_header("Content-Type", content_type); + content_length_ = in_length; + if (in_length > 0) { content_provider_ = std::move(provider); } + content_provider_resource_releaser_ = std::move(resource_releaser); + is_chunked_content_provider_ = false; +} + +inline void Response::set_content_provider( + const std::string &content_type, ContentProviderWithoutLength provider, + ContentProviderResourceReleaser resource_releaser) { + set_header("Content-Type", content_type); + content_length_ = 0; + content_provider_ = detail::ContentProviderAdapter(std::move(provider)); + content_provider_resource_releaser_ = std::move(resource_releaser); + is_chunked_content_provider_ = false; +} + +inline void Response::set_chunked_content_provider( + const std::string &content_type, ContentProviderWithoutLength provider, + ContentProviderResourceReleaser resource_releaser) { + set_header("Content-Type", content_type); + content_length_ = 0; + content_provider_ = detail::ContentProviderAdapter(std::move(provider)); + content_provider_resource_releaser_ = std::move(resource_releaser); + is_chunked_content_provider_ = true; +} + +inline void Response::set_file_content(const std::string &path, + const std::string &content_type) { + file_content_path_ = path; + file_content_content_type_ = content_type; +} + +inline void Response::set_file_content(const std::string &path) { + file_content_path_ = path; +} + +// Result implementation +inline size_t Result::get_request_header_value_u64(const std::string &key, + size_t def, + size_t id) const { + return detail::get_header_value_u64(request_headers_, key, def, id); +} + +inline bool Result::has_request_header(const std::string &key) const { + return request_headers_.find(key) != request_headers_.end(); +} + +inline std::string Result::get_request_header_value(const std::string &key, + const char *def, + size_t id) const { + return detail::get_header_value(request_headers_, key, def, id); +} + +inline size_t +Result::get_request_header_value_count(const std::string &key) const { + auto r = request_headers_.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +// Stream implementation +inline ssize_t Stream::write(const char *ptr) { + return write(ptr, strlen(ptr)); +} + +inline ssize_t Stream::write(const std::string &s) { + return write(s.data(), s.size()); +} + +// BodyReader implementation +inline ssize_t detail::BodyReader::read(char *buf, size_t len) { + if (!stream) { + last_error = Error::Connection; + return -1; + } + if (eof) { return 0; } + + if (!chunked) { + // Content-Length based reading + if (has_content_length && bytes_read >= content_length) { + eof = true; + return 0; + } + + auto to_read = len; + if (has_content_length) { + auto remaining = content_length - bytes_read; + to_read = (std::min)(len, remaining); + } + auto n = stream->read(buf, to_read); + + if (n < 0) { + last_error = stream->get_error(); + if (last_error == Error::Success) { last_error = Error::Read; } + eof = true; + return n; + } + if (n == 0) { + // Unexpected EOF before content_length + last_error = stream->get_error(); + if (last_error == Error::Success) { last_error = Error::Read; } + eof = true; + return 0; + } + + bytes_read += static_cast(n); + if (has_content_length && bytes_read >= content_length) { eof = true; } + if (payload_max_length > 0 && bytes_read > payload_max_length) { + last_error = Error::ExceedMaxPayloadSize; + eof = true; + return -1; + } + return n; + } + + // Chunked transfer encoding: delegate to shared decoder instance. + if (!chunked_decoder) { chunked_decoder.reset(new ChunkedDecoder(*stream)); } + + size_t chunk_offset = 0; + size_t chunk_total = 0; + auto n = chunked_decoder->read_payload(buf, len, chunk_offset, chunk_total); + if (n < 0) { + last_error = stream->get_error(); + if (last_error == Error::Success) { last_error = Error::Read; } + eof = true; + return n; + } + + if (n == 0) { + // Final chunk observed. Leave trailer parsing to the caller (StreamHandle). + eof = true; + return 0; + } + + bytes_read += static_cast(n); + if (payload_max_length > 0 && bytes_read > payload_max_length) { + last_error = Error::ExceedMaxPayloadSize; + eof = true; + return -1; + } + return n; +} + +// ThreadPool implementation +inline ThreadPool::ThreadPool(size_t n, size_t max_n, size_t mqr) + : base_thread_count_(n), max_queued_requests_(mqr), idle_thread_count_(0), + shutdown_(false) { +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + if (max_n != 0 && max_n < n) { + std::string msg = "max_threads must be >= base_threads"; + throw std::invalid_argument(msg); + } +#endif + max_thread_count_ = max_n == 0 ? n : max_n; + threads_.reserve(base_thread_count_); +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + try { +#endif + for (size_t i = 0; i < base_thread_count_; i++) { + threads_.emplace_back(std::thread([this]() { worker(false); })); + } +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + } catch (...) { + // If thread creation fails partway (e.g., pthread_create returns EAGAIN), + // signal the workers we already spawned to exit and join them so the + // vector destructor does not see joinable threads (which would call + // std::terminate). Then rethrow so the caller learns of the failure. + { + std::unique_lock lock(mutex_); + shutdown_ = true; + } + cond_.notify_all(); + for (auto &t : threads_) { + if (t.joinable()) { t.join(); } + } + throw; + } +#endif +} + +inline bool ThreadPool::enqueue(std::function fn) { + { + std::unique_lock lock(mutex_); + if (shutdown_) { return false; } + if (max_queued_requests_ > 0 && jobs_.size() >= max_queued_requests_) { + return false; + } + jobs_.push_back(std::move(fn)); + + // Spawn a dynamic thread if no idle threads and under max + if (idle_thread_count_ == 0 && + threads_.size() + dynamic_threads_.size() < max_thread_count_) { + cleanup_finished_threads(); + dynamic_threads_.emplace_back(std::thread([this]() { worker(true); })); + } + } + + cond_.notify_one(); + return true; +} + +inline void ThreadPool::shutdown() { + { + std::unique_lock lock(mutex_); + shutdown_ = true; + } + + cond_.notify_all(); + + for (auto &t : threads_) { + if (t.joinable()) { t.join(); } + } + + // Move dynamic_threads_ to a local list under the lock to avoid racing + // with worker threads that call move_to_finished() concurrently. + std::list remaining_dynamic; + { + std::unique_lock lock(mutex_); + remaining_dynamic = std::move(dynamic_threads_); + } + for (auto &t : remaining_dynamic) { + if (t.joinable()) { t.join(); } + } + + std::unique_lock lock(mutex_); + cleanup_finished_threads(); +} + +inline void ThreadPool::move_to_finished(std::thread::id id) { + // Must be called with mutex_ held + for (auto it = dynamic_threads_.begin(); it != dynamic_threads_.end(); ++it) { + if (it->get_id() == id) { + finished_threads_.push_back(std::move(*it)); + dynamic_threads_.erase(it); + return; + } + } +} + +inline void ThreadPool::cleanup_finished_threads() { + // Must be called with mutex_ held + for (auto &t : finished_threads_) { + if (t.joinable()) { t.join(); } + } + finished_threads_.clear(); +} + +inline void ThreadPool::worker(bool is_dynamic) { + for (;;) { + std::function fn; + { + std::unique_lock lock(mutex_); + idle_thread_count_++; + + if (is_dynamic) { + auto has_work = cond_.wait_for( + lock, std::chrono::seconds(CPPHTTPLIB_THREAD_POOL_IDLE_TIMEOUT), + [&] { return !jobs_.empty() || shutdown_; }); + if (!has_work) { + // Timed out with no work - exit this dynamic thread + idle_thread_count_--; + move_to_finished(std::this_thread::get_id()); + break; + } + } else { + cond_.wait(lock, [&] { return !jobs_.empty() || shutdown_; }); + } + + idle_thread_count_--; + + if (shutdown_ && jobs_.empty()) { break; } + + fn = std::move(jobs_.front()); + jobs_.pop_front(); + } + + assert(true == static_cast(fn)); + fn(); + } + +#if defined(CPPHTTPLIB_OPENSSL_SUPPORT) && !defined(OPENSSL_IS_BORINGSSL) && \ + !defined(LIBRESSL_VERSION_NUMBER) + OPENSSL_thread_stop(); +#endif +} + +/* + * Group 1 (continued): detail namespace - Stream implementations + */ + +namespace detail { + +inline void calc_actual_timeout(time_t max_timeout_msec, time_t duration_msec, + time_t timeout_sec, time_t timeout_usec, + time_t &actual_timeout_sec, + time_t &actual_timeout_usec) { + auto timeout_msec = (timeout_sec * 1000) + (timeout_usec / 1000); + + auto actual_timeout_msec = + (std::min)(max_timeout_msec - duration_msec, timeout_msec); + + if (actual_timeout_msec < 0) { actual_timeout_msec = 0; } + + actual_timeout_sec = actual_timeout_msec / 1000; + actual_timeout_usec = (actual_timeout_msec % 1000) * 1000; +} + +// Socket stream implementation +inline SocketStream::SocketStream( + socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, time_t write_timeout_usec, + time_t max_timeout_msec, + std::chrono::time_point start_time) + : sock_(sock), read_timeout_sec_(read_timeout_sec), + read_timeout_usec_(read_timeout_usec), + write_timeout_sec_(write_timeout_sec), + write_timeout_usec_(write_timeout_usec), + max_timeout_msec_(max_timeout_msec), start_time_(start_time), + read_buff_(read_buff_size_, 0) {} + +inline SocketStream::~SocketStream() = default; + +inline bool SocketStream::is_readable() const { + return read_buff_off_ < read_buff_content_size_; +} + +inline bool SocketStream::wait_readable() const { + if (max_timeout_msec_ <= 0) { + return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0; + } + + time_t read_timeout_sec; + time_t read_timeout_usec; + calc_actual_timeout(max_timeout_msec_, duration(), read_timeout_sec_, + read_timeout_usec_, read_timeout_sec, read_timeout_usec); + + return select_read(sock_, read_timeout_sec, read_timeout_usec) > 0; +} + +inline bool SocketStream::wait_writable() const { + return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0; +} + +inline bool SocketStream::is_peer_alive() const { + return detail::is_socket_alive(sock_); +} + +inline ssize_t SocketStream::read(char *ptr, size_t size) { +#ifdef _WIN32 + size = + (std::min)(size, static_cast((std::numeric_limits::max)())); +#else + size = (std::min)(size, + static_cast((std::numeric_limits::max)())); +#endif + + if (read_buff_off_ < read_buff_content_size_) { + auto remaining_size = read_buff_content_size_ - read_buff_off_; + if (size <= remaining_size) { + memcpy(ptr, read_buff_.data() + read_buff_off_, size); + read_buff_off_ += size; + return static_cast(size); + } else { + memcpy(ptr, read_buff_.data() + read_buff_off_, remaining_size); + read_buff_off_ += remaining_size; + return static_cast(remaining_size); + } + } + + if (!wait_readable()) { + error_ = Error::Timeout; + return -1; + } + + read_buff_off_ = 0; + read_buff_content_size_ = 0; + + if (size < read_buff_size_) { + auto n = read_socket(sock_, read_buff_.data(), read_buff_size_, + CPPHTTPLIB_RECV_FLAGS); + if (n <= 0) { + if (n == 0) { + error_ = Error::ConnectionClosed; + } else { + error_ = Error::Read; + } + return n; + } else if (n <= static_cast(size)) { + memcpy(ptr, read_buff_.data(), static_cast(n)); + return n; + } else { + memcpy(ptr, read_buff_.data(), size); + read_buff_off_ = size; + read_buff_content_size_ = static_cast(n); + return static_cast(size); + } + } else { + auto n = read_socket(sock_, ptr, size, CPPHTTPLIB_RECV_FLAGS); + if (n <= 0) { + if (n == 0) { + error_ = Error::ConnectionClosed; + } else { + error_ = Error::Read; + } + } + return n; + } +} + +inline ssize_t SocketStream::write(const char *ptr, size_t size) { + if (!wait_writable()) { return -1; } + +#if defined(_WIN32) && !defined(_WIN64) + size = + (std::min)(size, static_cast((std::numeric_limits::max)())); +#endif + + return send_socket(sock_, ptr, size, CPPHTTPLIB_SEND_FLAGS); +} + +inline void SocketStream::get_remote_ip_and_port(std::string &ip, + int &port) const { + return detail::get_remote_ip_and_port(sock_, ip, port); +} + +inline void SocketStream::get_local_ip_and_port(std::string &ip, + int &port) const { + return detail::get_local_ip_and_port(sock_, ip, port); +} + +inline socket_t SocketStream::socket() const { return sock_; } + +inline time_t SocketStream::duration() const { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_time_) + .count(); +} + +inline void SocketStream::set_read_timeout(time_t sec, time_t usec) { + read_timeout_sec_ = sec; + read_timeout_usec_ = usec; +} + +// Buffer stream implementation +inline bool BufferStream::is_readable() const { return true; } + +inline bool BufferStream::wait_readable() const { return true; } + +inline bool BufferStream::wait_writable() const { return true; } + +inline ssize_t BufferStream::read(char *ptr, size_t size) { +#if defined(_MSC_VER) && _MSC_VER < 1910 + auto len_read = buffer._Copy_s(ptr, size, size, position); +#else + auto len_read = buffer.copy(ptr, size, position); +#endif + position += static_cast(len_read); + return static_cast(len_read); +} + +inline ssize_t BufferStream::write(const char *ptr, size_t size) { + buffer.append(ptr, size); + return static_cast(size); +} + +inline void BufferStream::get_remote_ip_and_port(std::string & /*ip*/, + int & /*port*/) const {} + +inline void BufferStream::get_local_ip_and_port(std::string & /*ip*/, + int & /*port*/) const {} + +inline socket_t BufferStream::socket() const { return 0; } + +inline time_t BufferStream::duration() const { return 0; } + +inline const std::string &BufferStream::get_buffer() const { return buffer; } + +inline PathParamsMatcher::PathParamsMatcher(const std::string &pattern) + : MatcherBase(pattern) { + constexpr const char marker[] = "/:"; + + // One past the last ending position of a path param substring + std::size_t last_param_end = 0; + +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + // Needed to ensure that parameter names are unique during matcher + // construction + // If exceptions are disabled, only last duplicate path + // parameter will be set + std::unordered_set param_name_set; +#endif + + while (true) { + const auto marker_pos = pattern.find( + marker, last_param_end == 0 ? last_param_end : last_param_end - 1); + if (marker_pos == std::string::npos) { break; } + + static_fragments_.push_back( + pattern.substr(last_param_end, marker_pos - last_param_end + 1)); + + const auto param_name_start = marker_pos + str_len(marker); + + auto sep_pos = pattern.find(separator, param_name_start); + if (sep_pos == std::string::npos) { sep_pos = pattern.length(); } + + auto param_name = + pattern.substr(param_name_start, sep_pos - param_name_start); + +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + if (param_name_set.find(param_name) != param_name_set.cend()) { + std::string msg = "Encountered path parameter '" + param_name + + "' multiple times in route pattern '" + pattern + "'."; + throw std::invalid_argument(msg); + } +#endif + + param_names_.push_back(std::move(param_name)); + + last_param_end = sep_pos + 1; + } + + if (last_param_end < pattern.length()) { + static_fragments_.push_back(pattern.substr(last_param_end)); + } +} + +inline bool PathParamsMatcher::match(Request &request) const { + request.matches = std::smatch(); + request.path_params.clear(); + request.path_params.reserve(param_names_.size()); + + // One past the position at which the path matched the pattern last time + std::size_t starting_pos = 0; + for (size_t i = 0; i < static_fragments_.size(); ++i) { + const auto &fragment = static_fragments_[i]; + + if (starting_pos + fragment.length() > request.path.length()) { + return false; + } + + // Avoid unnecessary allocation by using strncmp instead of substr + + // comparison + if (std::strncmp(request.path.c_str() + starting_pos, fragment.c_str(), + fragment.length()) != 0) { + return false; + } + + starting_pos += fragment.length(); + + // Should only happen when we have a static fragment after a param + // Example: '/users/:id/subscriptions' + // The 'subscriptions' fragment here does not have a corresponding param + if (i >= param_names_.size()) { continue; } + + auto sep_pos = request.path.find(separator, starting_pos); + if (sep_pos == std::string::npos) { sep_pos = request.path.length(); } + + const auto ¶m_name = param_names_[i]; + + request.path_params.emplace( + param_name, request.path.substr(starting_pos, sep_pos - starting_pos)); + + // Mark everything up to '/' as matched + starting_pos = sep_pos + 1; + } + // Returns false if the path is longer than the pattern + return starting_pos >= request.path.length(); +} + +inline bool RegexMatcher::match(Request &request) const { + request.path_params.clear(); + return std::regex_match(request.path, request.matches, regex_); +} + +// Enclose IPv6 address in brackets if needed +inline std::string prepare_host_string(const std::string &host) { + // Enclose IPv6 address in brackets (but not if already enclosed) + if (host.find(':') == std::string::npos || + (!host.empty() && host[0] == '[')) { + // IPv4, hostname, or already bracketed IPv6 + return host; + } else { + // IPv6 address without brackets + return "[" + host + "]"; + } +} + +inline std::string make_host_and_port_string(const std::string &host, int port, + bool is_ssl) { + auto result = prepare_host_string(host); + + // Append port if not default + if ((!is_ssl && port == 80) || (is_ssl && port == 443)) { + ; // do nothing + } else { + result += ":" + std::to_string(port); + } + + return result; +} + +// Create "host:port" string always including port number (for CONNECT method) +inline std::string +make_host_and_port_string_always_port(const std::string &host, int port) { + return prepare_host_string(host) + ":" + std::to_string(port); +} + +bool parse_no_proxy_entry(const std::string &token, NoProxyEntry &out); +NormalizedTarget normalize_target(const std::string &host); +bool ip_in_cidr(const IPBytes &ip, const IPBytes &net, int prefix_bits); +bool host_matches_no_proxy(const NormalizedTarget &target, + const std::vector &entries); + +inline bool ip_in_cidr(const IPBytes &ip, const IPBytes &net, int prefix_bits) { + if (prefix_bits < 0 || prefix_bits > 128) { return false; } + if (prefix_bits == 0) { return true; } + int full_bytes = prefix_bits / 8; + int rem_bits = prefix_bits % 8; + if (full_bytes > 0 && std::memcmp(ip.data(), net.data(), + static_cast(full_bytes)) != 0) { + return false; + } + if (rem_bits == 0) { return true; } + auto i = static_cast(full_bytes); + auto mask = static_cast(0xFFu << (8 - rem_bits)); + return (ip[i] & mask) == (net[i] & mask); +} + +inline bool parse_no_proxy_entry(const std::string &token, NoProxyEntry &out) { + if (token.empty()) { return false; } + + if (token == "*") { + out.kind = NoProxyKind::Wildcard; + return true; + } + + auto slash = token.find('/'); + std::string addr_part = + (slash == std::string::npos) ? token : token.substr(0, slash); + std::string prefix_part = + (slash == std::string::npos) ? std::string() : token.substr(slash + 1); + + // A bare slash or trailing-slash CIDR like "10.0.0.0/" is malformed; + // don't silently treat it as a /32 (or /128). + if (slash != std::string::npos && prefix_part.empty()) { return false; } + + // Accept the bracketed IPv6 form ("[::1]", "[fe80::]/10") as well as the + // bare form. Brackets have no meaning for IPv4, so skip the IPv4 attempt + // when brackets are present. + bool bracketed = addr_part.size() >= 2 && addr_part.front() == '[' && + addr_part.back() == ']'; + if (bracketed) { addr_part = addr_part.substr(1, addr_part.size() - 2); } + + if (!bracketed) { + struct in_addr v4; + if (inet_pton(AF_INET, addr_part.c_str(), &v4) == 1) { + int prefix = 32; + if (!prefix_part.empty()) { + auto r = from_chars(prefix_part.data(), + prefix_part.data() + prefix_part.size(), prefix); + if (r.ec != std::errc{} || + r.ptr != prefix_part.data() + prefix_part.size()) { + return false; + } + if (prefix < 0 || prefix > 32) { return false; } + } + out.kind = NoProxyKind::IPv4Cidr; + std::memcpy(out.net.data(), &v4, sizeof(v4)); + out.prefix_bits = prefix; + return true; + } + } + + struct in6_addr v6; + if (inet_pton(AF_INET6, addr_part.c_str(), &v6) == 1) { + int prefix = 128; + if (!prefix_part.empty()) { + auto r = from_chars(prefix_part.data(), + prefix_part.data() + prefix_part.size(), prefix); + if (r.ec != std::errc{} || + r.ptr != prefix_part.data() + prefix_part.size()) { + return false; + } + if (prefix < 0 || prefix > 128) { return false; } + } + out.kind = NoProxyKind::IPv6Cidr; + std::memcpy(out.net.data(), &v6, sizeof(v6)); + out.prefix_bits = prefix; + return true; + } + + // Bracketed entries can only be IPv6. If the IPv6 parse above failed, + // the entry is malformed — don't fall through to the hostname branch. + if (bracketed) { return false; } + + // A '/' on a non-IP token means a CIDR prefix without an address. Reject. + if (slash != std::string::npos) { return false; } + // Port-specific entries (host:port) are not supported. + if (token.find(':') != std::string::npos) { return false; } + + std::string hostname = case_ignore::to_lower(token); + while (!hostname.empty() && hostname.front() == '.') { + hostname.erase(hostname.begin()); + } + while (!hostname.empty() && hostname.back() == '.') { + hostname.pop_back(); + } + if (hostname.empty()) { return false; } + + out.kind = NoProxyKind::HostnameSuffix; + out.hostname_pattern = std::move(hostname); + return true; +} + +inline NormalizedTarget normalize_target(const std::string &host) { + NormalizedTarget t; + std::string h = host; + + if (h.size() >= 2 && h.front() == '[' && h.back() == ']') { + h = h.substr(1, h.size() - 2); + } + + // Strip a single trailing dot so "example.com." canonicalizes to + // "example.com". + if (!h.empty() && h.back() == '.') { h.pop_back(); } + + t.hostname = case_ignore::to_lower(h); + + if (!t.hostname.empty()) { + struct in_addr v4; + struct in6_addr v6; + if (inet_pton(AF_INET, t.hostname.c_str(), &v4) == 1) { + t.is_ipv4 = true; + std::memcpy(t.ip.data(), &v4, sizeof(v4)); + } else if (inet_pton(AF_INET6, t.hostname.c_str(), &v6) == 1) { + t.is_ipv6 = true; + std::memcpy(t.ip.data(), &v6, sizeof(v6)); + } + } + return t; +} + +inline bool host_matches_no_proxy(const NormalizedTarget &target, + const std::vector &entries) { + if (target.hostname.empty()) { return false; } + for (const auto &e : entries) { + switch (e.kind) { + case NoProxyKind::Wildcard: return true; + case NoProxyKind::IPv4Cidr: + if (target.is_ipv4 && ip_in_cidr(target.ip, e.net, e.prefix_bits)) { + return true; + } + break; + case NoProxyKind::IPv6Cidr: + if (target.is_ipv6 && ip_in_cidr(target.ip, e.net, e.prefix_bits)) { + return true; + } + break; + case NoProxyKind::HostnameSuffix: + if (target.is_ipv4 || target.is_ipv6) { break; } + if (target.hostname == e.hostname_pattern) { return true; } + // Dot-boundary suffix match: prevents "evilexample.com" from matching + // an entry of "example.com". + if (target.hostname.size() > e.hostname_pattern.size() + 1) { + auto offset = target.hostname.size() - e.hostname_pattern.size(); + if (target.hostname[offset - 1] == '.' && + target.hostname.compare(offset, e.hostname_pattern.size(), + e.hostname_pattern) == 0) { + return true; + } + } + break; + } + } + return false; +} + +template +inline bool check_and_write_headers(Stream &strm, Headers &headers, + T header_writer, Error &error) { + for (const auto &h : headers) { + if (!detail::fields::is_field_name(h.first) || + !detail::fields::is_field_value(h.second)) { + error = Error::InvalidHeaders; + return false; + } + } + if (header_writer(strm, headers) <= 0) { + error = Error::Write; + return false; + } + return true; +} + +} // namespace detail + +/* + * Group 2 (continued): detail namespace - SSLSocketStream implementation + */ + +#ifdef CPPHTTPLIB_SSL_ENABLED +namespace detail { + +// SSL socket stream implementation +inline SSLSocketStream::SSLSocketStream( + socket_t sock, tls::session_t session, time_t read_timeout_sec, + time_t read_timeout_usec, time_t write_timeout_sec, + time_t write_timeout_usec, time_t max_timeout_msec, + std::chrono::time_point start_time) + : sock_(sock), session_(session), read_timeout_sec_(read_timeout_sec), + read_timeout_usec_(read_timeout_usec), + write_timeout_sec_(write_timeout_sec), + write_timeout_usec_(write_timeout_usec), + max_timeout_msec_(max_timeout_msec), start_time_(start_time) { +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + // Clear AUTO_RETRY for proper non-blocking I/O timeout handling + // Note: create_session() also clears this, but SSLClient currently + // uses ssl_new() which does not. Until full TLS API migration is complete, + // we need to ensure AUTO_RETRY is cleared here regardless of how the + // SSL session was created. + SSL_clear_mode(static_cast(session), SSL_MODE_AUTO_RETRY); +#endif +} + +inline SSLSocketStream::~SSLSocketStream() = default; + +inline bool SSLSocketStream::is_readable() const { + return tls::pending(session_) > 0; +} + +inline bool SSLSocketStream::wait_readable() const { + if (max_timeout_msec_ <= 0) { + return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0; + } + + time_t read_timeout_sec; + time_t read_timeout_usec; + calc_actual_timeout(max_timeout_msec_, duration(), read_timeout_sec_, + read_timeout_usec_, read_timeout_sec, read_timeout_usec); + + return select_read(sock_, read_timeout_sec, read_timeout_usec) > 0; +} + +inline bool SSLSocketStream::wait_writable() const { + return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 && + !tls::is_peer_closed(session_, sock_); +} + +inline bool SSLSocketStream::is_peer_alive() const { + return !tls::is_peer_closed(session_, sock_); +} + +inline ssize_t SSLSocketStream::read(char *ptr, size_t size) { + if (tls::pending(session_) > 0) { + tls::TlsError err; + auto ret = tls::read(session_, ptr, size, err); + if (ret == 0 || err.code == tls::ErrorCode::PeerClosed) { + error_ = Error::ConnectionClosed; + } + return ret; + } else if (wait_readable()) { + tls::TlsError err; + auto ret = tls::read(session_, ptr, size, err); + if (ret < 0) { + auto n = 1000; +#ifdef _WIN32 + while (--n >= 0 && (err.code == tls::ErrorCode::WantRead || + (err.code == tls::ErrorCode::SyscallError && + WSAGetLastError() == WSAETIMEDOUT))) { +#else + while (--n >= 0 && err.code == tls::ErrorCode::WantRead) { +#endif + if (tls::pending(session_) > 0) { + return tls::read(session_, ptr, size, err); + } else if (wait_readable()) { + std::this_thread::sleep_for(std::chrono::microseconds{10}); + ret = tls::read(session_, ptr, size, err); + if (ret >= 0) { return ret; } + } else { + break; + } + } + assert(ret < 0); + } else if (ret == 0 || err.code == tls::ErrorCode::PeerClosed) { + error_ = Error::ConnectionClosed; + } + return ret; + } else { + error_ = Error::Timeout; + return -1; + } +} + +inline ssize_t SSLSocketStream::write(const char *ptr, size_t size) { + if (wait_writable()) { + auto handle_size = + std::min(size, (std::numeric_limits::max)()); + + tls::TlsError err; + auto ret = tls::write(session_, ptr, handle_size, err); + if (ret < 0) { + auto n = 1000; +#ifdef _WIN32 + while (--n >= 0 && (err.code == tls::ErrorCode::WantWrite || + (err.code == tls::ErrorCode::SyscallError && + WSAGetLastError() == WSAETIMEDOUT))) { +#else + while (--n >= 0 && err.code == tls::ErrorCode::WantWrite) { +#endif + if (wait_writable()) { + std::this_thread::sleep_for(std::chrono::microseconds{10}); + ret = tls::write(session_, ptr, handle_size, err); + if (ret >= 0) { return ret; } + } else { + break; + } + } + assert(ret < 0); + } + return ret; + } + return -1; +} + +inline void SSLSocketStream::get_remote_ip_and_port(std::string &ip, + int &port) const { + detail::get_remote_ip_and_port(sock_, ip, port); +} + +inline void SSLSocketStream::get_local_ip_and_port(std::string &ip, + int &port) const { + detail::get_local_ip_and_port(sock_, ip, port); +} + +inline socket_t SSLSocketStream::socket() const { return sock_; } + +inline time_t SSLSocketStream::duration() const { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_time_) + .count(); +} + +inline void SSLSocketStream::set_read_timeout(time_t sec, time_t usec) { + read_timeout_sec_ = sec; + read_timeout_usec_ = usec; +} + +} // namespace detail +#endif // CPPHTTPLIB_SSL_ENABLED + +/* + * Group 4: Server implementation + */ + +// HTTP server implementation +inline Server::Server() + : new_task_queue([] { + return new ThreadPool(CPPHTTPLIB_THREAD_POOL_COUNT, + CPPHTTPLIB_THREAD_POOL_MAX_COUNT); + }) { +#ifndef _WIN32 + signal(SIGPIPE, SIG_IGN); +#endif +} + +inline Server::~Server() = default; + +inline std::unique_ptr +Server::make_matcher(const std::string &pattern) { + if (pattern.find("/:") != std::string::npos) { + return detail::make_unique(pattern); + } else { + return detail::make_unique(pattern); + } +} + +inline Server &Server::Get(const std::string &pattern, Handler handler) { + return add_handler(get_handlers_, pattern, std::move(handler)); +} + +inline Server &Server::Post(const std::string &pattern, Handler handler) { + return add_handler(post_handlers_, pattern, std::move(handler)); +} + +inline Server &Server::Post(const std::string &pattern, + HandlerWithContentReader handler) { + return add_handler(post_handlers_for_content_reader_, pattern, + std::move(handler)); +} + +inline Server &Server::Put(const std::string &pattern, Handler handler) { + return add_handler(put_handlers_, pattern, std::move(handler)); +} + +inline Server &Server::Put(const std::string &pattern, + HandlerWithContentReader handler) { + return add_handler(put_handlers_for_content_reader_, pattern, + std::move(handler)); +} + +inline Server &Server::Patch(const std::string &pattern, Handler handler) { + return add_handler(patch_handlers_, pattern, std::move(handler)); +} + +inline Server &Server::Patch(const std::string &pattern, + HandlerWithContentReader handler) { + return add_handler(patch_handlers_for_content_reader_, pattern, + std::move(handler)); +} + +inline Server &Server::Delete(const std::string &pattern, Handler handler) { + return add_handler(delete_handlers_, pattern, std::move(handler)); +} + +inline Server &Server::Delete(const std::string &pattern, + HandlerWithContentReader handler) { + return add_handler(delete_handlers_for_content_reader_, pattern, + std::move(handler)); +} + +inline Server &Server::Options(const std::string &pattern, Handler handler) { + return add_handler(options_handlers_, pattern, std::move(handler)); +} + +inline Server &Server::WebSocket(const std::string &pattern, + WebSocketHandler handler) { + websocket_handlers_.push_back( + {make_matcher(pattern), std::move(handler), nullptr}); + return *this; +} + +inline Server &Server::WebSocket(const std::string &pattern, + WebSocketHandler handler, + SubProtocolSelector sub_protocol_selector) { + websocket_handlers_.push_back({make_matcher(pattern), std::move(handler), + std::move(sub_protocol_selector)}); + return *this; +} + +inline bool Server::set_base_dir(const std::string &dir, + const std::string &mount_point) { + return set_mount_point(mount_point, dir); +} + +inline bool Server::set_mount_point(const std::string &mount_point, + const std::string &dir, Headers headers) { + detail::FileStat stat(dir); + if (stat.is_dir()) { + std::string mnt = !mount_point.empty() ? mount_point : "/"; + if (!mnt.empty() && mnt[0] == '/') { + std::string resolved_base; + if (detail::canonicalize_path(dir.c_str(), resolved_base)) { +#if defined(_WIN32) + if (resolved_base.back() != '\\' && resolved_base.back() != '/') { + resolved_base += '\\'; + } +#else + if (resolved_base.back() != '/') { resolved_base += '/'; } +#endif + } + base_dirs_.push_back( + {std::move(mnt), dir, std::move(resolved_base), std::move(headers)}); + return true; + } + } + return false; +} + +inline bool Server::remove_mount_point(const std::string &mount_point) { + for (auto it = base_dirs_.begin(); it != base_dirs_.end(); ++it) { + if (it->mount_point == mount_point) { + base_dirs_.erase(it); + return true; + } + } + return false; +} + +inline Server & +Server::set_file_extension_and_mimetype_mapping(const std::string &ext, + const std::string &mime) { + file_extension_and_mimetype_map_[ext] = mime; + return *this; +} + +inline Server &Server::set_default_file_mimetype(const std::string &mime) { + default_file_mimetype_ = mime; + return *this; +} + +inline Server &Server::set_file_request_handler(Handler handler) { + file_request_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_error_handler_core(HandlerWithResponse handler, + std::true_type) { + error_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_error_handler_core(Handler handler, + std::false_type) { + error_handler_ = [handler](const Request &req, Response &res) { + handler(req, res); + return HandlerResponse::Handled; + }; + return *this; +} + +inline Server &Server::set_exception_handler(ExceptionHandler handler) { + exception_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_pre_routing_handler(HandlerWithResponse handler) { + pre_routing_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_post_routing_handler(Handler handler) { + post_routing_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_pre_request_handler(HandlerWithResponse handler) { + pre_request_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_logger(Logger logger) { + logger_ = std::move(logger); + return *this; +} + +inline Server &Server::set_error_logger(ErrorLogger error_logger) { + error_logger_ = std::move(error_logger); + return *this; +} + +inline Server &Server::set_pre_compression_logger(Logger logger) { + pre_compression_logger_ = std::move(logger); + return *this; +} + +inline Server & +Server::set_expect_100_continue_handler(Expect100ContinueHandler handler) { + expect_100_continue_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_start_handler(StartHandler handler) { + start_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_address_family(int family) { + address_family_ = family; + return *this; +} + +inline Server &Server::set_tcp_nodelay(bool on) { + tcp_nodelay_ = on; + return *this; +} + +inline Server &Server::set_ipv6_v6only(bool on) { + ipv6_v6only_ = on; + return *this; +} + +inline Server &Server::set_socket_options(SocketOptions socket_options) { + socket_options_ = std::move(socket_options); + return *this; +} + +inline Server &Server::set_default_headers(Headers headers) { + default_headers_ = std::move(headers); + return *this; +} + +inline Server &Server::set_header_writer( + std::function const &writer) { + header_writer_ = writer; + return *this; +} + +inline Server & +Server::set_trusted_proxies(const std::vector &proxies) { + trusted_proxies_ = proxies; + return *this; +} + +inline Server &Server::set_keep_alive_max_count(size_t count) { + keep_alive_max_count_ = count; + return *this; +} + +inline Server &Server::set_keep_alive_timeout(time_t sec) { + keep_alive_timeout_sec_ = sec; + return *this; +} + +template +inline Server &Server::set_keep_alive_timeout( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec(duration, [&](time_t sec, time_t /*usec*/) { + set_keep_alive_timeout(sec); + }); + return *this; +} + +inline Server &Server::set_read_timeout(time_t sec, time_t usec) { + read_timeout_sec_ = sec; + read_timeout_usec_ = usec; + return *this; +} + +inline Server &Server::set_write_timeout(time_t sec, time_t usec) { + write_timeout_sec_ = sec; + write_timeout_usec_ = usec; + return *this; +} + +inline Server &Server::set_idle_interval(time_t sec, time_t usec) { + idle_interval_sec_ = sec; + idle_interval_usec_ = usec; + return *this; +} + +inline Server &Server::set_payload_max_length(size_t length) { + payload_max_length_ = length; + return *this; +} + +inline Server &Server::set_websocket_max_missed_pongs(int count) { + websocket_max_missed_pongs_ = count; + return *this; +} + +inline Server &Server::set_websocket_ping_interval(time_t sec) { + websocket_ping_interval_sec_ = sec; + return *this; +} + +template +inline Server &Server::set_websocket_ping_interval( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec(duration, [&](time_t sec, time_t /*usec*/) { + set_websocket_ping_interval(sec); + }); + return *this; +} + +inline bool Server::bind_to_port(const std::string &host, int port, + int socket_flags) { + auto ret = bind_internal(host, port, socket_flags); + if (ret == -1) { is_decommissioned = true; } + return ret >= 0; +} +inline int Server::bind_to_any_port(const std::string &host, int socket_flags) { + auto ret = bind_internal(host, 0, socket_flags); + if (ret == -1) { is_decommissioned = true; } + return ret; +} + +inline bool Server::listen_after_bind() { return listen_internal(); } + +inline bool Server::listen(const std::string &host, int port, + int socket_flags) { + return bind_to_port(host, port, socket_flags) && listen_internal(); +} + +inline bool Server::is_running() const { return is_running_; } + +inline void Server::wait_until_ready() const { + while (!is_running_ && !is_decommissioned) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } +} + +inline void Server::stop() noexcept { + if (is_running_) { + assert(svr_sock_ != INVALID_SOCKET); + std::atomic sock(svr_sock_.exchange(INVALID_SOCKET)); + detail::shutdown_socket(sock); + detail::close_socket(sock); + } + is_decommissioned = false; +} + +inline void Server::decommission() { is_decommissioned = true; } + +inline bool Server::parse_request_line(const char *s, Request &req) const { + auto len = strlen(s); + if (len < 2 || s[len - 2] != '\r' || s[len - 1] != '\n') { return false; } + len -= 2; + + { + size_t count = 0; + + detail::split(s, s + len, ' ', [&](const char *b, const char *e) { + switch (count) { + case 0: req.method = std::string(b, e); break; + case 1: req.target = std::string(b, e); break; + case 2: req.version = std::string(b, e); break; + default: break; + } + count++; + }); + + if (count != 3) { return false; } + } + + thread_local const std::set methods{ + "GET", "HEAD", "POST", "PUT", "DELETE", + "CONNECT", "OPTIONS", "TRACE", "PATCH", "PRI"}; + + if (methods.find(req.method) == methods.end()) { + output_error_log(Error::InvalidHTTPMethod, &req); + return false; + } + + if (req.version != "HTTP/1.1" && req.version != "HTTP/1.0") { + output_error_log(Error::InvalidHTTPVersion, &req); + return false; + } + + { + // Skip URL fragment + for (size_t i = 0; i < req.target.size(); i++) { + if (req.target[i] == '#') { + req.target.erase(i); + break; + } + } + + detail::divide(req.target, '?', + [&](const char *lhs_data, std::size_t lhs_size, + const char *rhs_data, std::size_t rhs_size) { + req.path = + decode_path_component(std::string(lhs_data, lhs_size)); + detail::parse_query_text(rhs_data, rhs_size, req.params); + }); + } + + return true; +} + +inline bool Server::write_response(Stream &strm, bool close_connection, + Request &req, Response &res) { + // NOTE: `req.ranges` should be empty, otherwise it will be applied + // incorrectly to the error content. + req.ranges.clear(); + return write_response_core(strm, close_connection, req, res, false); +} + +inline bool Server::write_response_with_content(Stream &strm, + bool close_connection, + const Request &req, + Response &res) { + return write_response_core(strm, close_connection, req, res, true); +} + +inline bool Server::write_response_core(Stream &strm, bool close_connection, + const Request &req, Response &res, + bool need_apply_ranges) { + assert(res.status != -1); + + if (400 <= res.status && error_handler_ && + error_handler_(req, res) == HandlerResponse::Handled) { + need_apply_ranges = true; + } + + std::string content_type; + std::string boundary; + if (need_apply_ranges) { apply_ranges(req, res, content_type, boundary); } + + // Prepare additional headers + if (close_connection || req.get_header_value("Connection") == "close" || + 400 <= res.status) { // Don't leave connections open after errors + res.set_header("Connection", "close"); + } else { + std::string s = "timeout="; + s += std::to_string(keep_alive_timeout_sec_); + s += ", max="; + s += std::to_string(keep_alive_max_count_); + res.set_header("Keep-Alive", s); + } + + if ((!res.body.empty() || res.content_length_ > 0 || res.content_provider_) && + !res.has_header("Content-Type")) { + res.set_header("Content-Type", "text/plain"); + } + + if (res.body.empty() && !res.content_length_ && !res.content_provider_ && + !res.has_header("Content-Length")) { + res.set_header("Content-Length", "0"); + } + + if (req.method == "HEAD" && !res.has_header("Accept-Ranges")) { + res.set_header("Accept-Ranges", "bytes"); + } + + if (post_routing_handler_) { post_routing_handler_(req, res); } + + // Response line and headers + detail::BufferStream bstrm; + if (!detail::write_response_line(bstrm, res.status)) { return false; } + if (header_writer_(bstrm, res.headers) <= 0) { return false; } + + // Combine small body with headers to reduce write syscalls + if (req.method != "HEAD" && !res.body.empty() && !res.content_provider_) { + bstrm.write(res.body.data(), res.body.size()); + } + + // Log before writing to avoid race condition with client-side code that + // accesses logger-captured data immediately after receiving the response. + output_log(req, res); + + // Flush buffer + auto &data = bstrm.get_buffer(); + if (!detail::write_data(strm, data.data(), data.size())) { return false; } + + // Streaming body + auto ret = true; + if (req.method != "HEAD" && res.content_provider_) { + if (write_content_with_provider(strm, req, res, boundary, content_type)) { + res.content_provider_success_ = true; + } else { + ret = false; + } + } + + return ret; +} + +inline bool +Server::write_content_with_provider(Stream &strm, const Request &req, + Response &res, const std::string &boundary, + const std::string &content_type) { + auto is_shutting_down = [this]() { + return this->svr_sock_ == INVALID_SOCKET; + }; + + if (res.content_length_ > 0) { + if (req.ranges.empty()) { + return detail::write_content(strm, res.content_provider_, 0, + res.content_length_, is_shutting_down); + } else if (req.ranges.size() == 1) { + auto offset_and_length = detail::get_range_offset_and_length( + req.ranges[0], res.content_length_); + + return detail::write_content(strm, res.content_provider_, + offset_and_length.first, + offset_and_length.second, is_shutting_down); + } else { + return detail::write_multipart_ranges_data( + strm, req, res, boundary, content_type, res.content_length_, + is_shutting_down); + } + } else { + if (res.is_chunked_content_provider_) { + auto type = detail::encoding_type(req, res); + + auto compressor = detail::make_compressor(type); + if (!compressor) { + compressor = detail::make_unique(); + } + + return detail::write_content_chunked(strm, res.content_provider_, + is_shutting_down, *compressor); + } else { + return detail::write_content_without_length(strm, res.content_provider_, + is_shutting_down); + } + } +} + +inline bool Server::read_content(Stream &strm, Request &req, Response &res) { + FormFields::iterator cur_field; + FormFiles::iterator cur_file; + auto is_text_field = false; + size_t count = 0; + if (read_content_core( + strm, req, res, + // Regular + [&](const char *buf, size_t n) { + // Prevent arithmetic overflow when checking sizes. + // Avoid computing (req.body.size() + n) directly because + // adding two unsigned `size_t` values can wrap around and + // produce a small result instead of indicating overflow. + // Instead, check using subtraction: ensure `n` does not + // exceed the remaining capacity `max_size() - size()`. + if (req.body.size() >= req.body.max_size() || + n > req.body.max_size() - req.body.size()) { + return false; + } + + // Limit decompressed body size to payload_max_length_ to protect + // against "zip bomb" attacks where a small compressed payload + // decompresses to a massive size. + if (payload_max_length_ > 0 && + (req.body.size() >= payload_max_length_ || + n > payload_max_length_ - req.body.size())) { + return false; + } + + req.body.append(buf, n); + return true; + }, + // Multipart FormData + [&](const FormData &file) { + if (count++ == CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT) { + output_error_log(Error::TooManyFormDataFiles, &req); + return false; + } + + if (file.filename.empty()) { + cur_field = req.form.fields.emplace( + file.name, FormField{file.name, file.content, file.headers}); + is_text_field = true; + } else { + cur_file = req.form.files.emplace(file.name, file); + is_text_field = false; + } + return true; + }, + [&](const char *buf, size_t n) { + if (is_text_field) { + auto &content = cur_field->second.content; + if (content.size() + n > content.max_size()) { return false; } + content.append(buf, n); + } else { + auto &content = cur_file->second.content; + if (content.size() + n > content.max_size()) { return false; } + content.append(buf, n); + } + return true; + })) { + const auto &content_type = req.get_header_value("Content-Type"); + if (detail::extract_media_type(content_type) == + "application/x-www-form-urlencoded") { + if (req.body.size() > CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH) { + res.status = StatusCode::PayloadTooLarge_413; // NOTE: should be 414? + output_error_log(Error::ExceedMaxPayloadSize, &req); + return false; + } + detail::parse_query_text(req.body, req.params); + } + return true; + } + return false; +} + +inline bool Server::read_content_with_content_receiver( + Stream &strm, Request &req, Response &res, ContentReceiver receiver, + FormDataHeader multipart_header, ContentReceiver multipart_receiver) { + return read_content_core(strm, req, res, std::move(receiver), + std::move(multipart_header), + std::move(multipart_receiver)); +} + +inline bool Server::read_content_core( + Stream &strm, Request &req, Response &res, ContentReceiver receiver, + FormDataHeader multipart_header, ContentReceiver multipart_receiver) const { + detail::FormDataParser multipart_form_data_parser; + ContentReceiverWithProgress out; + + if (req.is_multipart_form_data()) { + const auto &content_type = req.get_header_value("Content-Type"); + std::string boundary; + if (!detail::parse_multipart_boundary(content_type, boundary)) { + res.status = StatusCode::BadRequest_400; + output_error_log(Error::MultipartParsing, &req); + return false; + } + + multipart_form_data_parser.set_boundary(std::move(boundary)); + out = [&](const char *buf, size_t n, size_t /*off*/, size_t /*len*/) { + return multipart_form_data_parser.parse(buf, n, multipart_header, + multipart_receiver); + }; + } else { + out = [receiver](const char *buf, size_t n, size_t /*off*/, + size_t /*len*/) { return receiver(buf, n); }; + } + + // RFC 9112 §6: no Transfer-Encoding and no Content-Length means no body. + // For non-SSL builds we still scan non-persistent connections for stray + // body bytes so the payload limit is enforced (413). On keep-alive, + // pending bytes may be the next request (issue #2450), so skip. +#if !defined(CPPHTTPLIB_SSL_ENABLED) + if (!req.has_header("Content-Length") && + !detail::is_chunked_transfer_encoding(req.headers)) { + if (!detail::is_connection_persistent(req) && payload_max_length_ > 0 && + payload_max_length_ < (std::numeric_limits::max)()) { + auto has_data = strm.is_readable(); + if (!has_data) { + auto s = strm.socket(); + if (s != INVALID_SOCKET) { + has_data = detail::select_read(s, 0, 0) > 0; + } + } + if (has_data) { + auto result = + detail::read_content_without_length(strm, payload_max_length_, out); + if (result == detail::ReadContentResult::PayloadTooLarge) { + res.status = StatusCode::PayloadTooLarge_413; + return false; + } else if (result != detail::ReadContentResult::Success) { + return false; + } + return true; + } + } + return true; + } +#else + if (!req.has_header("Content-Length") && + !detail::is_chunked_transfer_encoding(req.headers)) { + return true; + } +#endif + + if (!detail::read_content(strm, req, payload_max_length_, res.status, nullptr, + out, true)) { + return false; + } + + req.body_consumed_ = true; + + if (req.is_multipart_form_data()) { + if (!multipart_form_data_parser.is_valid()) { + res.status = StatusCode::BadRequest_400; + output_error_log(Error::MultipartParsing, &req); + return false; + } + } + + return true; +} + +inline bool Server::handle_file_request(Request &req, Response &res) { + for (const auto &entry : base_dirs_) { + // Prefix match + if (!req.path.compare(0, entry.mount_point.size(), entry.mount_point)) { + std::string sub_path = "/" + req.path.substr(entry.mount_point.size()); + if (detail::is_valid_path(sub_path)) { + auto path = entry.base_dir + sub_path; + if (path.back() == '/') { path += "index.html"; } + + // Defense-in-depth: is_valid_path blocks ".." traversal in the URL, + // but symlinks/junctions can still escape the base directory. + if (!entry.resolved_base_dir.empty()) { + std::string resolved_path; + if (detail::canonicalize_path(path.c_str(), resolved_path) && + !detail::is_path_within_base(resolved_path, + entry.resolved_base_dir)) { + res.status = StatusCode::Forbidden_403; + return true; + } + } + + detail::FileStat stat(path); + + if (stat.is_dir()) { + res.set_redirect(sub_path + "/", StatusCode::MovedPermanently_301); + return true; + } + + if (stat.is_file()) { + for (const auto &kv : entry.headers) { + res.set_header(kv.first, kv.second); + } + + auto etag = detail::compute_etag(stat); + if (!etag.empty()) { res.set_header("ETag", etag); } + + auto mtime = stat.mtime(); + + auto last_modified = detail::file_mtime_to_http_date(mtime); + if (!last_modified.empty()) { + res.set_header("Last-Modified", last_modified); + } + + if (check_if_not_modified(req, res, etag, mtime)) { return true; } + + check_if_range(req, etag, mtime); + + auto mm = std::make_shared(path.c_str()); + if (!mm->is_open()) { + output_error_log(Error::OpenFile, &req); + return false; + } + + res.set_content_provider( + mm->size(), + detail::find_content_type(path, file_extension_and_mimetype_map_, + default_file_mimetype_), + [mm](size_t offset, size_t length, DataSink &sink) -> bool { + sink.write(mm->data() + offset, length); + return true; + }); + + if (req.method != "HEAD" && file_request_handler_) { + file_request_handler_(req, res); + } + + return true; + } else { + output_error_log(Error::OpenFile, &req); + } + } + } + } + return false; +} + +inline bool Server::check_if_not_modified(const Request &req, Response &res, + const std::string &etag, + time_t mtime) const { + // Handle conditional GET: + // 1. If-None-Match takes precedence (RFC 9110 Section 13.1.2) + // 2. If-Modified-Since is checked only when If-None-Match is absent + if (req.has_header("If-None-Match")) { + if (!etag.empty()) { + auto val = req.get_header_value("If-None-Match"); + + // NOTE: We use exact string matching here. This works correctly + // because our server always generates weak ETags (W/"..."), and + // clients typically send back the same ETag they received. + // RFC 9110 Section 8.8.3.2 allows weak comparison for + // If-None-Match, where W/"x" and "x" would match, but this + // simplified implementation requires exact matches. + auto ret = detail::split_find(val.data(), val.data() + val.size(), ',', + [&](const char *b, const char *e) { + auto seg_len = static_cast(e - b); + return (seg_len == 1 && *b == '*') || + (seg_len == etag.size() && + std::equal(b, e, etag.begin())); + }); + + if (ret) { + res.status = StatusCode::NotModified_304; + return true; + } + } + } else if (req.has_header("If-Modified-Since")) { + auto val = req.get_header_value("If-Modified-Since"); + auto t = detail::parse_http_date(val); + + if (t != static_cast(-1) && mtime <= t) { + res.status = StatusCode::NotModified_304; + return true; + } + } + return false; +} + +inline bool Server::check_if_range(Request &req, const std::string &etag, + time_t mtime) const { + // Handle If-Range for partial content requests (RFC 9110 + // Section 13.1.5). If-Range is only evaluated when Range header is + // present. If the validator matches, serve partial content; otherwise + // serve full content. + if (!req.ranges.empty() && req.has_header("If-Range")) { + auto val = req.get_header_value("If-Range"); + + auto is_valid_range = [&]() { + if (detail::is_strong_etag(val)) { + // RFC 9110 Section 13.1.5: If-Range requires strong ETag + // comparison. + return (!etag.empty() && val == etag); + } else if (detail::is_weak_etag(val)) { + // Weak ETags are not valid for If-Range (RFC 9110 Section 13.1.5) + return false; + } else { + // HTTP-date comparison + auto t = detail::parse_http_date(val); + return (t != static_cast(-1) && mtime <= t); + } + }; + + if (!is_valid_range()) { + // Validator doesn't match: ignore Range and serve full content + req.ranges.clear(); + return false; + } + } + + return true; +} + +inline socket_t +Server::create_server_socket(const std::string &host, int port, + int socket_flags, + SocketOptions socket_options) const { + return detail::create_socket( + host, std::string(), port, address_family_, socket_flags, tcp_nodelay_, + ipv6_v6only_, std::move(socket_options), + [&](socket_t sock, struct addrinfo &ai, bool & /*quit*/) -> bool { + if (::bind(sock, ai.ai_addr, static_cast(ai.ai_addrlen))) { + output_error_log(Error::BindIPAddress, nullptr); + return false; + } + if (::listen(sock, CPPHTTPLIB_LISTEN_BACKLOG)) { + output_error_log(Error::Listen, nullptr); + return false; + } + return true; + }); +} + +inline int Server::bind_internal(const std::string &host, int port, + int socket_flags) { + if (is_decommissioned) { return -1; } + + if (!is_valid()) { return -1; } + + svr_sock_ = create_server_socket(host, port, socket_flags, socket_options_); + if (svr_sock_ == INVALID_SOCKET) { return -1; } + + if (port == 0) { + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + if (getsockname(svr_sock_, reinterpret_cast(&addr), + &addr_len) == -1) { + output_error_log(Error::GetSockName, nullptr); + return -1; + } + if (addr.ss_family == AF_INET) { + return ntohs(reinterpret_cast(&addr)->sin_port); + } else if (addr.ss_family == AF_INET6) { + return ntohs(reinterpret_cast(&addr)->sin6_port); + } else { + output_error_log(Error::UnsupportedAddressFamily, nullptr); + return -1; + } + } else { + return port; + } +} + +inline bool Server::listen_internal() { + if (is_decommissioned) { return false; } + + auto ret = true; + is_running_ = true; + auto se = detail::scope_exit([&]() { is_running_ = false; }); + + if (start_handler_) { start_handler_(); } + + { + std::unique_ptr task_queue(new_task_queue()); + + while (svr_sock_ != INVALID_SOCKET) { +#ifndef _WIN32 + if (idle_interval_sec_ > 0 || idle_interval_usec_ > 0) { +#endif + auto val = detail::select_read(svr_sock_, idle_interval_sec_, + idle_interval_usec_); + if (val == 0) { // Timeout + task_queue->on_idle(); + continue; + } +#ifndef _WIN32 + } +#endif + +#if defined _WIN32 + // sockets connected via WASAccept inherit flags NO_HANDLE_INHERIT, + // OVERLAPPED + socket_t sock = WSAAccept(svr_sock_, nullptr, nullptr, nullptr, 0); +#elif defined SOCK_CLOEXEC + socket_t sock = accept4(svr_sock_, nullptr, nullptr, SOCK_CLOEXEC); +#else + socket_t sock = accept(svr_sock_, nullptr, nullptr); +#endif + + if (sock == INVALID_SOCKET) { + if (errno == EMFILE) { + // The per-process limit of open file descriptors has been reached. + // Try to accept new connections after a short sleep. + std::this_thread::sleep_for(std::chrono::microseconds{1}); + continue; + } else if (errno == EINTR || errno == EAGAIN) { + continue; + } + if (svr_sock_ != INVALID_SOCKET) { + detail::close_socket(svr_sock_); + ret = false; + output_error_log(Error::Connection, nullptr); + } else { + ; // The server socket was closed by user. + } + break; + } + + detail::set_socket_opt_time(sock, SOL_SOCKET, SO_RCVTIMEO, + read_timeout_sec_, read_timeout_usec_); + detail::set_socket_opt_time(sock, SOL_SOCKET, SO_SNDTIMEO, + write_timeout_sec_, write_timeout_usec_); + + if (tcp_nodelay_) { set_socket_opt(sock, IPPROTO_TCP, TCP_NODELAY, 1); } + + if (!task_queue->enqueue( + [this, sock]() { process_and_close_socket(sock); })) { + output_error_log(Error::ResourceExhaustion, nullptr); + detail::shutdown_socket(sock); + detail::close_socket(sock); + } + } + + task_queue->shutdown(); + } + + is_decommissioned = !ret; + return ret; +} + +inline bool Server::routing(Request &req, Response &res, Stream &strm) { + if (pre_routing_handler_ && + pre_routing_handler_(req, res) == HandlerResponse::Handled) { + return true; + } + + // File handler + if ((req.method == "GET" || req.method == "HEAD") && + handle_file_request(req, res)) { + return true; + } + + if (detail::expect_content(req)) { + // Content reader handler + { + // Track whether the ContentReader was aborted due to the decompressed + // payload exceeding `payload_max_length_`. + // The user handler runs after the lambda returns, so we must restore the + // 413 status if the handler overwrites it. + bool content_reader_payload_too_large = false; + + ContentReader reader( + [&](ContentReceiver receiver) { + auto result = read_content_with_content_receiver( + strm, req, res, std::move(receiver), nullptr, nullptr); + if (!result) { + output_error_log(Error::Read, &req); + if (res.status == StatusCode::PayloadTooLarge_413) { + content_reader_payload_too_large = true; + } + } + return result; + }, + [&](FormDataHeader header, ContentReceiver receiver) { + auto result = read_content_with_content_receiver( + strm, req, res, nullptr, std::move(header), + std::move(receiver)); + if (!result) { + output_error_log(Error::Read, &req); + if (res.status == StatusCode::PayloadTooLarge_413) { + content_reader_payload_too_large = true; + } + } + return result; + }); + + bool dispatched = false; + if (req.method == "POST") { + dispatched = dispatch_request_for_content_reader( + req, res, std::move(reader), post_handlers_for_content_reader_); + } else if (req.method == "PUT") { + dispatched = dispatch_request_for_content_reader( + req, res, std::move(reader), put_handlers_for_content_reader_); + } else if (req.method == "PATCH") { + dispatched = dispatch_request_for_content_reader( + req, res, std::move(reader), patch_handlers_for_content_reader_); + } else if (req.method == "DELETE") { + dispatched = dispatch_request_for_content_reader( + req, res, std::move(reader), delete_handlers_for_content_reader_); + } + + if (dispatched) { + if (content_reader_payload_too_large) { + // Enforce the limit: override any status the handler may have set + // and return false so the error path sends a plain 413 response. + res.status = StatusCode::PayloadTooLarge_413; + res.body.clear(); + res.content_length_ = 0; + res.content_provider_ = nullptr; + return false; + } + return true; + } + } + + // NOTE: `req.body` is not read here. For a regular handler the body is + // read inside dispatch_request(), after the route has matched and the + // pre-request handler has approved the request, so that a rejected + // request (e.g. failed authentication) never forces us to buffer a + // potentially large body. + } + + // Regular handler + if (req.method == "GET" || req.method == "HEAD") { + return dispatch_request(req, res, get_handlers_, strm); + } else if (req.method == "POST") { + return dispatch_request(req, res, post_handlers_, strm); + } else if (req.method == "PUT") { + return dispatch_request(req, res, put_handlers_, strm); + } else if (req.method == "DELETE") { + return dispatch_request(req, res, delete_handlers_, strm); + } else if (req.method == "OPTIONS") { + return dispatch_request(req, res, options_handlers_, strm); + } else if (req.method == "PATCH") { + return dispatch_request(req, res, patch_handlers_, strm); + } + + res.status = StatusCode::BadRequest_400; + return false; +} + +inline bool Server::dispatch_request(Request &req, Response &res, + const Handlers &handlers, Stream &strm) { + for (const auto &x : handlers) { + const auto &matcher = x.first; + const auto &handler = x.second; + + if (matcher->match(req)) { + req.matched_route = matcher->pattern(); + + // Run the pre-request handler before reading the body so a rejected + // request (e.g. failed authentication) never forces us to buffer a + // potentially large body. `req.matched_route` is available here. + if (pre_request_handler_ && + pre_request_handler_(req, res) == HandlerResponse::Handled) { + return true; + } + + // The route matched and the request was approved; read the body now. + if (detail::expect_content(req) && !read_content(strm, req, res)) { + output_error_log(Error::Read, &req); + return false; + } + + handler(req, res); + return true; + } + } + return false; +} + +inline void Server::apply_ranges(const Request &req, Response &res, + std::string &content_type, + std::string &boundary) const { + if (req.ranges.size() > 1 && res.status == StatusCode::PartialContent_206) { + auto it = res.headers.find("Content-Type"); + if (it != res.headers.end()) { + content_type = it->second; + res.headers.erase(it); + } + + boundary = detail::make_multipart_data_boundary(); + + res.set_header("Content-Type", + "multipart/byteranges; boundary=" + boundary); + } + + auto type = detail::encoding_type(req, res); + + if (res.body.empty()) { + if (res.content_length_ > 0) { + size_t length = 0; + if (req.ranges.empty() || res.status != StatusCode::PartialContent_206) { + length = res.content_length_; + } else if (req.ranges.size() == 1) { + auto offset_and_length = detail::get_range_offset_and_length( + req.ranges[0], res.content_length_); + + length = offset_and_length.second; + + auto content_range = detail::make_content_range_header_field( + offset_and_length, res.content_length_); + res.set_header("Content-Range", content_range); + } else { + length = detail::get_multipart_ranges_data_length( + req, boundary, content_type, res.content_length_); + } + res.set_header("Content-Length", std::to_string(length)); + } else { + if (res.content_provider_) { + if (res.is_chunked_content_provider_) { + res.set_header("Transfer-Encoding", "chunked"); + if (type != detail::EncodingType::None) { + res.set_header("Content-Encoding", detail::encoding_name(type)); + res.set_header("Vary", "Accept-Encoding"); + } + } + } + } + } else { + if (req.ranges.empty() || res.status != StatusCode::PartialContent_206) { + ; + } else if (req.ranges.size() == 1) { + auto offset_and_length = + detail::get_range_offset_and_length(req.ranges[0], res.body.size()); + auto offset = offset_and_length.first; + auto length = offset_and_length.second; + + auto content_range = detail::make_content_range_header_field( + offset_and_length, res.body.size()); + res.set_header("Content-Range", content_range); + + assert(offset + length <= res.body.size()); + res.body = res.body.substr(offset, length); + } else { + std::string data; + detail::make_multipart_ranges_data(req, res, boundary, content_type, + res.body.size(), data); + res.body.swap(data); + } + + if (type != detail::EncodingType::None) { + output_pre_compression_log(req, res); + + if (auto compressor = detail::make_compressor(type)) { + std::string compressed; + if (compressor->compress(res.body.data(), res.body.size(), true, + [&](const char *data, size_t data_len) { + compressed.append(data, data_len); + return true; + })) { + res.body.swap(compressed); + res.set_header("Content-Encoding", detail::encoding_name(type)); + res.set_header("Vary", "Accept-Encoding"); + } + } + } + + auto length = std::to_string(res.body.size()); + res.set_header("Content-Length", length); + } +} + +inline bool Server::dispatch_request_for_content_reader( + Request &req, Response &res, ContentReader content_reader, + const HandlersForContentReader &handlers) const { + for (const auto &x : handlers) { + const auto &matcher = x.first; + const auto &handler = x.second; + + if (matcher->match(req)) { + req.matched_route = matcher->pattern(); + if (!pre_request_handler_ || + pre_request_handler_(req, res) != HandlerResponse::Handled) { + handler(req, res, content_reader); + } + return true; + } + } + return false; +} + +inline std::string +get_client_ip(const std::string &x_forwarded_for, + const std::vector &trusted_proxies) { + // X-Forwarded-For is a comma-separated list per RFC 7239 + std::vector ip_list; + detail::split(x_forwarded_for.data(), + x_forwarded_for.data() + x_forwarded_for.size(), ',', + [&](const char *b, const char *e) { + auto r = detail::trim(b, e, 0, static_cast(e - b)); + ip_list.emplace_back(std::string(b + r.first, b + r.second)); + }); + + // A malformed X-Forwarded-For (empty, comma-only, whitespace-only) yields + // no segments. Signal "no client IP derived" with an empty string so the + // caller can fall back to the connection-level remote address. + if (ip_list.empty()) { return std::string(); } + + for (size_t i = 0; i < ip_list.size(); ++i) { + auto ip = ip_list[i]; + + auto is_trusted_proxy = + std::any_of(trusted_proxies.begin(), trusted_proxies.end(), + [&](const std::string &proxy) { return ip == proxy; }); + + if (is_trusted_proxy) { + if (i == 0) { + // If the trusted proxy is the first IP, there's no preceding client IP + return ip; + } else { + // Return the IP immediately before the trusted proxy + return ip_list[i - 1]; + } + } + } + + // If no trusted proxy is found, return the first IP in the list + return ip_list.front(); +} + +inline bool +Server::process_request(Stream &strm, const std::string &remote_addr, + int remote_port, const std::string &local_addr, + int local_port, bool close_connection, + bool &connection_closed, + const std::function &setup_request, + bool *websocket_upgraded) { + std::array buf{}; + + detail::stream_line_reader line_reader(strm, buf.data(), buf.size()); + + // Connection has been closed on client + if (!line_reader.getline()) { return false; } + + Request req; + req.start_time_ = std::chrono::steady_clock::now(); + req.remote_addr = remote_addr; + req.remote_port = remote_port; + req.local_addr = local_addr; + req.local_port = local_port; + + Response res; + res.version = "HTTP/1.1"; + res.headers = default_headers_; + + // Request line and headers + if (!parse_request_line(line_reader.ptr(), req)) { + res.status = StatusCode::BadRequest_400; + output_error_log(Error::InvalidRequestLine, &req); + return write_response(strm, close_connection, req, res); + } + + // Request headers + if (!detail::read_headers(strm, req.headers)) { + res.status = StatusCode::BadRequest_400; + output_error_log(Error::InvalidHeaders, &req); + return write_response(strm, close_connection, req, res); + } + + // RFC 9112 §6.3: Reject requests with both a non-zero Content-Length and + // any Transfer-Encoding to prevent request smuggling. Content-Length: 0 is + // tolerated for compatibility with existing clients. + if (req.get_header_value_u64("Content-Length") > 0 && + req.has_header("Transfer-Encoding")) { + connection_closed = true; + res.status = StatusCode::BadRequest_400; + return write_response(strm, close_connection, req, res); + } + + // Check if the request URI doesn't exceed the limit + if (req.target.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) { + connection_closed = true; + res.status = StatusCode::UriTooLong_414; + output_error_log(Error::ExceedUriMaxLength, &req); + return write_response(strm, close_connection, req, res); + } + + if (req.get_header_value("Connection") == "close") { + connection_closed = true; + } + + if (req.version == "HTTP/1.0" && + req.get_header_value("Connection") != "Keep-Alive") { + connection_closed = true; + } + + if (!trusted_proxies_.empty() && req.has_header("X-Forwarded-For")) { + auto x_forwarded_for = req.get_header_value("X-Forwarded-For"); + auto derived = get_client_ip(x_forwarded_for, trusted_proxies_); + req.remote_addr = derived.empty() ? remote_addr : derived; + } else { + req.remote_addr = remote_addr; + } + req.remote_port = remote_port; + + req.local_addr = local_addr; + req.local_port = local_port; + + if (req.has_header("Accept")) { + const auto &accept_header = req.get_header_value("Accept"); + if (!detail::parse_accept_header(accept_header, req.accept_content_types)) { + connection_closed = true; + res.status = StatusCode::BadRequest_400; + output_error_log(Error::HTTPParsing, &req); + return write_response(strm, close_connection, req, res); + } + } + + if (req.has_header("Range")) { + const auto &range_header_value = req.get_header_value("Range"); + if (!detail::parse_range_header(range_header_value, req.ranges)) { + connection_closed = true; + res.status = StatusCode::RangeNotSatisfiable_416; + output_error_log(Error::InvalidRangeHeader, &req); + return write_response(strm, close_connection, req, res); + } + } + + if (setup_request) { setup_request(req); } + + if (req.get_header_value("Expect") == "100-continue") { + int status = StatusCode::Continue_100; + if (expect_100_continue_handler_) { + status = expect_100_continue_handler_(req, res); + } + switch (status) { + case StatusCode::Continue_100: + case StatusCode::ExpectationFailed_417: + detail::write_response_line(strm, status); + strm.write("\r\n"); + break; + default: + connection_closed = true; + return write_response(strm, true, req, res); + } + } + + // Setup `is_connection_closed` method + auto sock = strm.socket(); + req.is_connection_closed = [sock]() { + return !detail::is_socket_alive(sock); + }; + + // WebSocket upgrade + // Check pre_routing_handler_ before upgrading so that authentication + // and other middleware can reject the request with an HTTP response + // (e.g., 401) before the protocol switches. + if (detail::is_websocket_upgrade(req)) { + if (pre_routing_handler_ && + pre_routing_handler_(req, res) == HandlerResponse::Handled) { + if (res.status == -1) { res.status = StatusCode::OK_200; } + return write_response(strm, close_connection, req, res); + } + // Find matching WebSocket handler + for (const auto &entry : websocket_handlers_) { + if (entry.matcher->match(req)) { + // Compute accept key + auto client_key = req.get_header_value("Sec-WebSocket-Key"); + auto accept_key = detail::websocket_accept_key(client_key); + + // Negotiate subprotocol + std::string selected_subprotocol; + if (entry.sub_protocol_selector) { + auto protocol_header = req.get_header_value("Sec-WebSocket-Protocol"); + if (!protocol_header.empty()) { + std::vector protocols; + std::istringstream iss(protocol_header); + std::string token; + while (std::getline(iss, token, ',')) { + // Trim whitespace + auto start = token.find_first_not_of(' '); + auto end = token.find_last_not_of(' '); + if (start != std::string::npos) { + protocols.push_back(token.substr(start, end - start + 1)); + } + } + selected_subprotocol = entry.sub_protocol_selector(protocols); + } + } + + // Send 101 Switching Protocols + std::string handshake_response = "HTTP/1.1 101 Switching Protocols\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Accept: " + + accept_key + "\r\n"; + if (!selected_subprotocol.empty()) { + if (!detail::fields::is_field_value(selected_subprotocol)) { + return false; + } + handshake_response += + "Sec-WebSocket-Protocol: " + selected_subprotocol + "\r\n"; + } + handshake_response += "\r\n"; + if (strm.write(handshake_response.data(), handshake_response.size()) < + 0) { + return false; + } + + connection_closed = true; + if (websocket_upgraded) { *websocket_upgraded = true; } + + { + // Use WebSocket-specific read timeout instead of HTTP timeout + strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0); + ws::WebSocket ws(strm, req, true, websocket_ping_interval_sec_, + websocket_max_missed_pongs_); + entry.handler(req, ws); + } + return true; + } + } + // No matching handler - fall through to 404 + } + + // Routing + auto routed = false; +#ifdef CPPHTTPLIB_NO_EXCEPTIONS + routed = routing(req, res, strm); +#else + try { + routed = routing(req, res, strm); + } catch (std::exception &) { + if (exception_handler_) { + auto ep = std::current_exception(); + exception_handler_(req, res, ep); + routed = true; + } else { + res.status = StatusCode::InternalServerError_500; + } + } catch (...) { + if (exception_handler_) { + auto ep = std::current_exception(); + exception_handler_(req, res, ep); + routed = true; + } else { + res.status = StatusCode::InternalServerError_500; + } + } +#endif + auto ret = false; + if (routed) { + if (res.status == -1) { + res.status = req.ranges.empty() ? StatusCode::OK_200 + : StatusCode::PartialContent_206; + } + + // Serve file content by using a content provider + auto file_open_error = false; + if (!res.file_content_path_.empty()) { + const auto &path = res.file_content_path_; + auto mm = std::make_shared(path.c_str()); + if (!mm->is_open()) { + res.body.clear(); + res.content_length_ = 0; + res.content_provider_ = nullptr; + res.status = StatusCode::NotFound_404; + output_error_log(Error::OpenFile, &req); + file_open_error = true; + } else { + auto content_type = res.file_content_content_type_; + if (content_type.empty()) { + content_type = detail::find_content_type( + path, file_extension_and_mimetype_map_, default_file_mimetype_); + } + + res.set_content_provider( + mm->size(), content_type, + [mm](size_t offset, size_t length, DataSink &sink) -> bool { + sink.write(mm->data() + offset, length); + return true; + }); + } + } + + if (file_open_error) { + ret = write_response(strm, close_connection, req, res); + } else if (detail::range_error(req, res)) { + res.body.clear(); + res.content_length_ = 0; + res.content_provider_ = nullptr; + res.status = StatusCode::RangeNotSatisfiable_416; + ret = write_response(strm, close_connection, req, res); + } else { + ret = write_response_with_content(strm, close_connection, req, res); + } + } else { + if (res.status == -1) { res.status = StatusCode::NotFound_404; } + ret = write_response(strm, close_connection, req, res); + } + + // Drain any unconsumed framed body to prevent request smuggling on + // keep-alive. Without framing there is no body to drain — reading would + // consume the next request (issue #2450). + if (!req.body_consumed_ && detail::has_framed_body(req)) { + int dummy_status; + if (!detail::read_content( + strm, req, payload_max_length_, dummy_status, nullptr, + [](const char *, size_t, size_t, size_t) { return true; }, false)) { + connection_closed = true; + } + } + + return ret; +} + +inline bool Server::is_valid() const { return true; } + +inline bool Server::process_and_close_socket(socket_t sock) { + std::string remote_addr; + int remote_port = 0; + detail::get_remote_ip_and_port(sock, remote_addr, remote_port); + + std::string local_addr; + int local_port = 0; + detail::get_local_ip_and_port(sock, local_addr, local_port); + + bool websocket_upgraded = false; + auto ret = detail::process_server_socket( + svr_sock_, sock, keep_alive_max_count_, keep_alive_timeout_sec_, + read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, + write_timeout_usec_, + [&](Stream &strm, bool close_connection, bool &connection_closed) { + return process_request(strm, remote_addr, remote_port, local_addr, + local_port, close_connection, connection_closed, + nullptr, &websocket_upgraded); + }); + + detail::shutdown_socket(sock); + detail::close_socket(sock); + return ret; +} + +inline void Server::output_log(const Request &req, const Response &res) const { + if (logger_) { + std::lock_guard guard(logger_mutex_); + logger_(req, res); + } +} + +inline void Server::output_pre_compression_log(const Request &req, + const Response &res) const { + if (pre_compression_logger_) { + std::lock_guard guard(logger_mutex_); + pre_compression_logger_(req, res); + } +} + +inline void Server::output_error_log(const Error &err, + const Request *req) const { + if (error_logger_) { + std::lock_guard guard(logger_mutex_); + error_logger_(err, req); + } +} + +/* + * Group 5: ClientImpl and Client (Universal) implementation + */ +// HTTP client implementation +inline ClientImpl::ClientImpl(const std::string &host) + : ClientImpl(host, 80, std::string(), std::string()) {} + +inline ClientImpl::ClientImpl(const std::string &host, int port) + : ClientImpl(host, port, std::string(), std::string()) {} + +inline ClientImpl::ClientImpl(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path) + : host_(detail::escape_abstract_namespace_unix_domain(host)), port_(port), + client_cert_path_(client_cert_path), client_key_path_(client_key_path) {} + +inline ClientImpl::~ClientImpl() { + // Wait until all the requests in flight are handled. + size_t retry_count = 10; + while (retry_count-- > 0) { + { + std::lock_guard guard(socket_mutex_); + if (socket_requests_in_flight_ == 0) { break; } + } + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + + std::lock_guard guard(socket_mutex_); + shutdown_socket(socket_); + close_socket(socket_); +} + +inline bool ClientImpl::is_valid() const { return true; } + +inline void ClientImpl::copy_settings(const ClientImpl &rhs) { + client_cert_path_ = rhs.client_cert_path_; + client_key_path_ = rhs.client_key_path_; + connection_timeout_sec_ = rhs.connection_timeout_sec_; + read_timeout_sec_ = rhs.read_timeout_sec_; + read_timeout_usec_ = rhs.read_timeout_usec_; + write_timeout_sec_ = rhs.write_timeout_sec_; + write_timeout_usec_ = rhs.write_timeout_usec_; + max_timeout_msec_ = rhs.max_timeout_msec_; + basic_auth_username_ = rhs.basic_auth_username_; + basic_auth_password_ = rhs.basic_auth_password_; + bearer_token_auth_token_ = rhs.bearer_token_auth_token_; + keep_alive_ = rhs.keep_alive_; + follow_location_ = rhs.follow_location_; + path_encode_ = rhs.path_encode_; + address_family_ = rhs.address_family_; + tcp_nodelay_ = rhs.tcp_nodelay_; + ipv6_v6only_ = rhs.ipv6_v6only_; + socket_options_ = rhs.socket_options_; + compress_ = rhs.compress_; + decompress_ = rhs.decompress_; + payload_max_length_ = rhs.payload_max_length_; + has_payload_max_length_ = rhs.has_payload_max_length_; + interface_ = rhs.interface_; + proxy_host_ = rhs.proxy_host_; + proxy_port_ = rhs.proxy_port_; + proxy_basic_auth_username_ = rhs.proxy_basic_auth_username_; + proxy_basic_auth_password_ = rhs.proxy_basic_auth_password_; + proxy_bearer_token_auth_token_ = rhs.proxy_bearer_token_auth_token_; + no_proxy_entries_ = rhs.no_proxy_entries_; + logger_ = rhs.logger_; + error_logger_ = rhs.error_logger_; + +#ifdef CPPHTTPLIB_SSL_ENABLED + digest_auth_username_ = rhs.digest_auth_username_; + digest_auth_password_ = rhs.digest_auth_password_; + proxy_digest_auth_username_ = rhs.proxy_digest_auth_username_; + proxy_digest_auth_password_ = rhs.proxy_digest_auth_password_; + ca_cert_file_path_ = rhs.ca_cert_file_path_; + ca_cert_dir_path_ = rhs.ca_cert_dir_path_; + server_certificate_verification_ = rhs.server_certificate_verification_; + server_hostname_verification_ = rhs.server_hostname_verification_; + system_ca_mode_ = rhs.system_ca_mode_; +#endif +} + +inline bool +ClientImpl::is_proxy_enabled_for_host(const std::string &host) const { + if (proxy_host_.empty() || proxy_port_ == -1) { return false; } + if (no_proxy_entries_.empty()) { return true; } + // host_ is const so its normalized form is invariant; cache it. The + // cross-host path (setup_redirect_client passing next_host) re-normalizes. + if (host == host_) { + if (!host_normalized_valid_) { + host_normalized_ = detail::normalize_target(host_); + host_normalized_valid_ = true; + } + return !detail::host_matches_no_proxy(host_normalized_, no_proxy_entries_); + } + auto target = detail::normalize_target(host); + return !detail::host_matches_no_proxy(target, no_proxy_entries_); +} + +inline socket_t ClientImpl::create_client_socket(Error &error) const { + if (is_proxy_enabled_for_host(host_)) { + return detail::create_client_socket( + proxy_host_, std::string(), proxy_port_, address_family_, tcp_nodelay_, + ipv6_v6only_, socket_options_, connection_timeout_sec_, + connection_timeout_usec_, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_, interface_, error); + } + + // Check is custom IP specified for host_ + std::string ip; + auto it = addr_map_.find(host_); + if (it != addr_map_.end()) { ip = it->second; } + + return detail::create_client_socket( + host_, ip, port_, address_family_, tcp_nodelay_, ipv6_v6only_, + socket_options_, connection_timeout_sec_, connection_timeout_usec_, + read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, + write_timeout_usec_, interface_, error); +} + +inline bool ClientImpl::create_and_connect_socket(Socket &socket, + Error &error) { + auto sock = create_client_socket(error); + if (sock == INVALID_SOCKET) { return false; } + socket.sock = sock; + return true; +} + +inline bool ClientImpl::ensure_socket_connection(Socket &socket, Error &error) { + return create_and_connect_socket(socket, error); +} + +inline bool ClientImpl::setup_proxy_connection( + Socket & /*socket*/, + std::chrono::time_point /*start_time*/, + Response & /*res*/, bool & /*success*/, Error & /*error*/) { + return true; +} + +inline void ClientImpl::shutdown_ssl(Socket & /*socket*/, + bool /*shutdown_gracefully*/) { + // If there are any requests in flight from threads other than us, then it's + // a thread-unsafe race because individual ssl* objects are not thread-safe. + assert(socket_requests_in_flight_ == 0 || + socket_requests_are_from_thread_ == std::this_thread::get_id()); +} + +inline void ClientImpl::shutdown_socket(Socket &socket) const { + if (socket.sock == INVALID_SOCKET) { return; } + detail::shutdown_socket(socket.sock); +} + +inline void ClientImpl::close_socket(Socket &socket) { + // If there are requests in flight in another thread, usually closing + // the socket will be fine and they will simply receive an error when + // using the closed socket, but it is still a bug since rarely the OS + // may reassign the socket id to be used for a new socket, and then + // suddenly they will be operating on a live socket that is different + // than the one they intended! + assert(socket_requests_in_flight_ == 0 || + socket_requests_are_from_thread_ == std::this_thread::get_id()); + + // It is also a bug if this happens while SSL is still active +#ifdef CPPHTTPLIB_SSL_ENABLED + assert(socket.ssl == nullptr); +#endif + + if (socket.sock == INVALID_SOCKET) { return; } + detail::close_socket(socket.sock); + socket.sock = INVALID_SOCKET; +} + +inline void ClientImpl::disconnect(bool gracefully) { + shutdown_ssl(socket_, gracefully); + shutdown_socket(socket_); + close_socket(socket_); +} + +inline bool ClientImpl::read_response_line(Stream &strm, const Request &req, + Response &res, + bool skip_100_continue) const { + std::array buf{}; + + detail::stream_line_reader line_reader(strm, buf.data(), buf.size()); + + if (!line_reader.getline()) { return false; } + +#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR + thread_local const std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n"); +#else + thread_local const std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n"); +#endif + + std::cmatch m; + if (!std::regex_match(line_reader.ptr(), m, re)) { + return req.method == "CONNECT"; + } + res.version = std::string(m[1]); + res.status = std::stoi(std::string(m[2])); + res.reason = std::string(m[3]); + + // Ignore '100 Continue' (only when not using Expect: 100-continue explicitly) + while (skip_100_continue && res.status == StatusCode::Continue_100) { + if (!line_reader.getline()) { return false; } // CRLF + if (!line_reader.getline()) { return false; } // next response line + + if (!std::regex_match(line_reader.ptr(), m, re)) { return false; } + res.version = std::string(m[1]); + res.status = std::stoi(std::string(m[2])); + res.reason = std::string(m[3]); + } + + return true; +} + +inline bool ClientImpl::send(Request &req, Response &res, Error &error) { + std::lock_guard request_mutex_guard(request_mutex_); + auto ret = send_(req, res, error); + if (error == Error::SSLPeerCouldBeClosed_) { + assert(!ret); + ret = send_(req, res, error); + // If still failing with SSLPeerCouldBeClosed_, convert to Read error + if (error == Error::SSLPeerCouldBeClosed_) { error = Error::Read; } + } + return ret; +} + +inline bool ClientImpl::send_(Request &req, Response &res, Error &error) { + { + std::lock_guard guard(socket_mutex_); + + // Set this to false immediately - if it ever gets set to true by the end + // of the request, we know another thread instructed us to close the + // socket. + socket_should_be_closed_when_request_is_done_ = false; + + auto is_alive = false; + if (socket_.is_open()) { + is_alive = detail::is_socket_alive(socket_.sock); + +#ifdef CPPHTTPLIB_SSL_ENABLED + if (is_alive && is_ssl()) { + if (tls::is_peer_closed(socket_.ssl, socket_.sock)) { + is_alive = false; + } + } +#endif + + if (!is_alive) { + // Peer seems gone — non-graceful shutdown to avoid SIGPIPE. + disconnect(/*gracefully=*/false); + } + } + + if (!is_alive) { + if (!ensure_socket_connection(socket_, error)) { + output_error_log(error, &req); + return false; + } + + { + auto success = true; + if (!setup_proxy_connection(socket_, req.start_time_, res, success, + error)) { + if (!success) { output_error_log(error, &req); } + return success; + } + } + } + + // Mark the current socket as being in use so that it cannot be closed by + // anyone else while this request is ongoing, even though we will be + // releasing the mutex. + if (socket_requests_in_flight_ > 1) { + assert(socket_requests_are_from_thread_ == std::this_thread::get_id()); + } + socket_requests_in_flight_ += 1; + socket_requests_are_from_thread_ = std::this_thread::get_id(); + } + + for (const auto &header : default_headers_) { + if (req.headers.find(header.first) == req.headers.end()) { + req.headers.insert(header); + } + } + + auto ret = false; + auto close_connection = !keep_alive_; + + auto se = detail::scope_exit([&]() { + // Briefly lock mutex in order to mark that a request is no longer ongoing + std::lock_guard guard(socket_mutex_); + socket_requests_in_flight_ -= 1; + if (socket_requests_in_flight_ <= 0) { + assert(socket_requests_in_flight_ == 0); + socket_requests_are_from_thread_ = std::thread::id(); + } + + if (socket_should_be_closed_when_request_is_done_ || close_connection || + !ret) { + disconnect(/*gracefully=*/true); + } + }); + + ret = process_socket(socket_, req.start_time_, [&](Stream &strm) { + return handle_request(strm, req, res, close_connection, error); + }); + + if (!ret) { + if (error == Error::Success) { + error = Error::Unknown; + output_error_log(error, &req); + } + } + + return ret; +} + +inline Result ClientImpl::send(const Request &req) { + auto req2 = req; + return send_(std::move(req2)); +} + +inline Result ClientImpl::send_(Request &&req) { + auto res = detail::make_unique(); + auto error = Error::Success; + auto ret = send(req, *res, error); +#ifdef CPPHTTPLIB_SSL_ENABLED + return Result{ret ? std::move(res) : nullptr, error, std::move(req.headers), + last_ssl_error_, last_backend_error_}; +#else + return Result{ret ? std::move(res) : nullptr, error, std::move(req.headers)}; +#endif +} + +inline void ClientImpl::prepare_default_headers(Request &r, bool for_stream, + const std::string &ct) { + (void)for_stream; + for (const auto &header : default_headers_) { + if (!r.has_header(header.first)) { r.headers.insert(header); } + } + + if (!r.has_header("Host")) { + if (address_family_ == AF_UNIX) { + r.headers.emplace("Host", "localhost"); + } else { + r.headers.emplace( + "Host", detail::make_host_and_port_string(host_, port_, is_ssl())); + } + } + + if (!r.has_header("Accept")) { r.headers.emplace("Accept", "*/*"); } + + if (!r.content_receiver) { + if (!r.has_header("Accept-Encoding")) { + std::string accept_encoding; +#ifdef CPPHTTPLIB_BROTLI_SUPPORT + accept_encoding = "br"; +#endif +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + if (!accept_encoding.empty()) { accept_encoding += ", "; } + accept_encoding += "gzip, deflate"; +#endif +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + if (!accept_encoding.empty()) { accept_encoding += ", "; } + accept_encoding += "zstd"; +#endif + r.set_header("Accept-Encoding", accept_encoding); + } + +#ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENT + if (!r.has_header("User-Agent")) { + auto agent = std::string("cpp-httplib/") + CPPHTTPLIB_VERSION; + r.set_header("User-Agent", agent); + } +#endif + } + + if (!r.body.empty()) { + if (!ct.empty() && !r.has_header("Content-Type")) { + r.headers.emplace("Content-Type", ct); + } + if (!r.has_header("Content-Length")) { + r.headers.emplace("Content-Length", std::to_string(r.body.size())); + } + } +} + +inline ClientImpl::StreamHandle +ClientImpl::open_stream(const std::string &method, const std::string &path, + const Params ¶ms, const Headers &headers, + const std::string &body, + const std::string &content_type) { + StreamHandle handle; + handle.response = detail::make_unique(); + handle.error = Error::Success; + + auto query_path = params.empty() ? path : append_query_params(path, params); + handle.connection_ = detail::make_unique(); + + { + std::lock_guard guard(socket_mutex_); + + auto is_alive = false; + if (socket_.is_open()) { + is_alive = detail::is_socket_alive(socket_.sock); +#ifdef CPPHTTPLIB_SSL_ENABLED + if (is_alive && is_ssl()) { + if (tls::is_peer_closed(socket_.ssl, socket_.sock)) { + is_alive = false; + } + } +#endif + if (!is_alive) { disconnect(/*gracefully=*/false); } + } + + if (!is_alive) { + if (!ensure_socket_connection(socket_, handle.error)) { + handle.response.reset(); + return handle; + } + + { + auto success = true; + auto start_time = std::chrono::steady_clock::now(); + if (!setup_proxy_connection(socket_, start_time, *handle.response, + success, handle.error)) { + if (!success) { handle.response.reset(); } + return handle; + } + } + } + + transfer_socket_ownership_to_handle(handle); + } + +#ifdef CPPHTTPLIB_SSL_ENABLED + if (is_ssl() && handle.connection_->session) { + handle.socket_stream_ = detail::make_unique( + handle.connection_->sock, handle.connection_->session, + read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, + write_timeout_usec_); + } else { + handle.socket_stream_ = detail::make_unique( + handle.connection_->sock, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_); + } +#else + handle.socket_stream_ = detail::make_unique( + handle.connection_->sock, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_); +#endif + handle.stream_ = handle.socket_stream_.get(); + + Request req; + req.method = method; + req.path = query_path; + req.headers = headers; + req.body = body; + + prepare_default_headers(req, true, content_type); + + auto &strm = *handle.stream_; + if (detail::write_request_line(strm, req.method, req.path) < 0) { + handle.error = Error::Write; + handle.response.reset(); + return handle; + } + + if (!detail::check_and_write_headers(strm, req.headers, header_writer_, + handle.error)) { + handle.response.reset(); + return handle; + } + + if (!body.empty()) { + if (strm.write(body.data(), body.size()) < 0) { + handle.error = Error::Write; + handle.response.reset(); + return handle; + } + } + + if (!read_response_line(strm, req, *handle.response) || + !detail::read_headers(strm, handle.response->headers)) { + handle.error = Error::Read; + handle.response.reset(); + return handle; + } + + handle.body_reader_.stream = handle.stream_; + handle.body_reader_.payload_max_length = payload_max_length_; + + if (handle.response->has_header("Content-Length")) { + bool is_invalid = false; + auto content_length = detail::get_header_value_u64( + handle.response->headers, "Content-Length", 0, 0, is_invalid); + if (is_invalid) { + handle.error = Error::Read; + handle.response.reset(); + return handle; + } + handle.body_reader_.has_content_length = true; + handle.body_reader_.content_length = content_length; + } + + auto transfer_encoding = + handle.response->get_header_value("Transfer-Encoding"); + handle.body_reader_.chunked = (transfer_encoding == "chunked"); + + auto content_encoding = handle.response->get_header_value("Content-Encoding"); + if (!content_encoding.empty()) { + handle.decompressor_ = detail::create_decompressor(content_encoding); + } + + return handle; +} + +inline ssize_t ClientImpl::StreamHandle::read(char *buf, size_t len) { + if (!is_valid() || !response) { return -1; } + + if (decompressor_) { return read_with_decompression(buf, len); } + auto n = detail::read_body_content(stream_, body_reader_, buf, len); + + if (n <= 0 && body_reader_.chunked && !trailers_parsed_ && stream_) { + trailers_parsed_ = true; + if (body_reader_.chunked_decoder) { + if (!body_reader_.chunked_decoder->parse_trailers_into( + response->trailers, response->headers)) { + return n; + } + } else { + detail::ChunkedDecoder dec(*stream_); + if (!dec.parse_trailers_into(response->trailers, response->headers)) { + return n; + } + } + } + + return n; +} + +inline ssize_t ClientImpl::StreamHandle::read_with_decompression(char *buf, + size_t len) { + if (decompress_offset_ < decompress_buffer_.size()) { + auto available = decompress_buffer_.size() - decompress_offset_; + auto to_copy = (std::min)(len, available); + std::memcpy(buf, decompress_buffer_.data() + decompress_offset_, to_copy); + decompress_offset_ += to_copy; + decompressed_bytes_read_ += to_copy; + return static_cast(to_copy); + } + + decompress_buffer_.clear(); + decompress_offset_ = 0; + + constexpr size_t kDecompressionBufferSize = 8192; + char compressed_buf[kDecompressionBufferSize]; + + while (true) { + auto n = detail::read_body_content(stream_, body_reader_, compressed_buf, + sizeof(compressed_buf)); + + if (n <= 0) { return n; } + + bool decompress_ok = decompressor_->decompress( + compressed_buf, static_cast(n), + [this](const char *data, size_t data_len) { + decompress_buffer_.append(data, data_len); + auto limit = body_reader_.payload_max_length; + if (decompressed_bytes_read_ + decompress_buffer_.size() > limit) { + return false; + } + return true; + }); + + if (!decompress_ok) { + body_reader_.last_error = Error::Read; + return -1; + } + + if (!decompress_buffer_.empty()) { break; } + } + + auto to_copy = (std::min)(len, decompress_buffer_.size()); + std::memcpy(buf, decompress_buffer_.data(), to_copy); + decompress_offset_ = to_copy; + decompressed_bytes_read_ += to_copy; + return static_cast(to_copy); +} + +inline void ClientImpl::StreamHandle::parse_trailers_if_needed() { + if (!response || !stream_ || !body_reader_.chunked || trailers_parsed_) { + return; + } + + trailers_parsed_ = true; + + const auto bufsiz = 128; + char line_buf[bufsiz]; + detail::stream_line_reader line_reader(*stream_, line_buf, bufsiz); + + if (!line_reader.getline()) { return; } + + if (!detail::parse_trailers(line_reader, response->trailers, + response->headers)) { + return; + } +} + +namespace detail { + +inline ChunkedDecoder::ChunkedDecoder(Stream &s) : strm(s) {} + +inline ssize_t ChunkedDecoder::read_payload(char *buf, size_t len, + size_t &out_chunk_offset, + size_t &out_chunk_total) { + if (finished) { return 0; } + + if (chunk_remaining == 0) { + stream_line_reader lr(strm, line_buf, sizeof(line_buf)); + if (!lr.getline()) { return -1; } + + // RFC 9112 §7.1: chunk-size = 1*HEXDIG + const char *p = lr.ptr(); + int v = 0; + if (!is_hex(*p, v)) { return -1; } + + size_t chunk_len = 0; + constexpr size_t chunk_len_max = (std::numeric_limits::max)(); + for (; is_hex(*p, v); ++p) { + if (chunk_len > (chunk_len_max >> 4)) { return -1; } + chunk_len = (chunk_len << 4) | static_cast(v); + } + + while (is_space_or_tab(*p)) { + ++p; + } + if (*p != '\0' && *p != ';' && *p != '\r' && *p != '\n') { return -1; } + + if (chunk_len == 0) { + chunk_remaining = 0; + finished = true; + out_chunk_offset = 0; + out_chunk_total = 0; + return 0; + } + + chunk_remaining = chunk_len; + last_chunk_total = chunk_remaining; + last_chunk_offset = 0; + } + + auto to_read = (std::min)(chunk_remaining, len); + auto n = strm.read(buf, to_read); + if (n <= 0) { return -1; } + + auto offset_before = last_chunk_offset; + last_chunk_offset += static_cast(n); + chunk_remaining -= static_cast(n); + + out_chunk_offset = offset_before; + out_chunk_total = last_chunk_total; + + if (chunk_remaining == 0) { + stream_line_reader lr(strm, line_buf, sizeof(line_buf)); + if (!lr.getline()) { return -1; } + if (std::strcmp(lr.ptr(), "\r\n") != 0) { return -1; } + } + + return n; +} + +inline bool ChunkedDecoder::parse_trailers_into(Headers &dest, + const Headers &src_headers) { + stream_line_reader lr(strm, line_buf, sizeof(line_buf)); + if (!lr.getline()) { return false; } + return parse_trailers(lr, dest, src_headers); +} + +} // namespace detail + +inline void +ClientImpl::transfer_socket_ownership_to_handle(StreamHandle &handle) { + handle.connection_->sock = socket_.sock; +#ifdef CPPHTTPLIB_SSL_ENABLED + handle.connection_->session = socket_.ssl; + socket_.ssl = nullptr; +#endif + socket_.sock = INVALID_SOCKET; +} + +inline bool ClientImpl::handle_request(Stream &strm, Request &req, + Response &res, bool close_connection, + Error &error) { + if (req.path.empty()) { + error = Error::Connection; + output_error_log(error, &req); + return false; + } + + auto req_save = req; + + bool ret; + + if (!is_ssl() && is_proxy_enabled_for_host(host_)) { + auto req2 = req; + req2.path = "http://" + + detail::make_host_and_port_string(host_, port_, false) + + req.path; + ret = process_request(strm, req2, res, close_connection, error); + req = std::move(req2); + req.path = req_save.path; + } else { + ret = process_request(strm, req, res, close_connection, error); + } + + if (!ret) { return false; } + + if (res.get_header_value("Connection") == "close" || + (res.version == "HTTP/1.0" && res.reason != "Connection established")) { + // NOTE: this requires a not-entirely-obvious chain of calls to be correct + // for this to be safe. + + // This is safe to call because handle_request is only called by send_ + // which locks the request mutex during the process. It would be a bug + // to call it from a different thread since it's a thread-safety issue + // to do these things to the socket if another thread is using the socket. + std::lock_guard guard(socket_mutex_); + disconnect(/*gracefully=*/true); + } + + if (300 < res.status && res.status < 400 && follow_location_) { + req = std::move(req_save); + ret = redirect(req, res, error); + } + +#ifdef CPPHTTPLIB_SSL_ENABLED + if ((res.status == StatusCode::Unauthorized_401 || + res.status == StatusCode::ProxyAuthenticationRequired_407) && + req.authorization_count_ < 5) { + auto is_proxy = res.status == StatusCode::ProxyAuthenticationRequired_407; + + // Only retry when the 407 actually came from a proxy hop: plain HTTP + // through an enabled proxy. HTTPS via CONNECT tunnels the 407 from the + // origin (#2457); direct/bypassed origins have no proxy hop at all. + if (is_proxy && !(!is_ssl() && is_proxy_enabled_for_host(host_))) { + return ret; + } + + const auto &username = + is_proxy ? proxy_digest_auth_username_ : digest_auth_username_; + const auto &password = + is_proxy ? proxy_digest_auth_password_ : digest_auth_password_; + + if (!username.empty() && !password.empty()) { + std::map auth; + if (detail::parse_www_authenticate(res, auth, is_proxy)) { + Request new_req = req; + new_req.authorization_count_ += 1; + new_req.headers.erase(is_proxy ? "Proxy-Authorization" + : "Authorization"); + new_req.headers.insert(detail::make_digest_authentication_header( + req, auth, new_req.authorization_count_, detail::random_string(10), + username, password, is_proxy)); + + Response new_res; + + ret = send(new_req, new_res, error); + if (ret) { res = std::move(new_res); } + } + } + } +#endif + + return ret; +} + +inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { + if (req.redirect_count_ == 0) { + error = Error::ExceedRedirectCount; + output_error_log(error, &req); + return false; + } + + auto location = res.get_header_value("location"); + if (location.empty()) { return false; } + + detail::UrlComponents uc; + if (!detail::parse_url(location, uc)) { return false; } + + // Only follow http/https redirects + if (!uc.scheme.empty() && uc.scheme != "http" && uc.scheme != "https") { + return false; + } + + auto scheme = is_ssl() ? "https" : "http"; + + auto next_scheme = std::move(uc.scheme); + auto next_host = std::move(uc.host); + auto port_str = std::move(uc.port); + auto next_path = std::move(uc.path); + auto next_query = std::move(uc.query); + + auto next_port = port_; + if (!port_str.empty()) { + if (!detail::parse_port(port_str, next_port)) { return false; } + } else if (!next_scheme.empty()) { + next_port = next_scheme == "https" ? 443 : 80; + } + + if (next_scheme.empty()) { next_scheme = scheme; } + if (next_host.empty()) { next_host = host_; } + if (next_path.empty()) { next_path = "/"; } + + auto path = decode_path_component(next_path) + next_query; + + // Same host redirect - use current client + if (next_scheme == scheme && next_host == host_ && next_port == port_) { + return detail::redirect(*this, req, res, path, location, error); + } + + // Cross-host/scheme redirect - create new client with robust setup + return create_redirect_client(next_scheme, next_host, next_port, req, res, + path, location, error); +} + +// New method for robust redirect client creation +inline bool ClientImpl::create_redirect_client( + const std::string &scheme, const std::string &host, int port, Request &req, + Response &res, const std::string &path, const std::string &location, + Error &error) { + // Determine if we need SSL + auto need_ssl = (scheme == "https"); + + // Clean up request headers that are host/client specific + // Remove headers that should not be carried over to new host + auto headers_to_remove = + std::vector{"Host", "Proxy-Authorization", "Authorization"}; + + for (const auto &header_name : headers_to_remove) { + auto it = req.headers.find(header_name); + while (it != req.headers.end()) { + it = req.headers.erase(it); + it = req.headers.find(header_name); + } + } + + // Create appropriate client type and handle redirect + if (need_ssl) { +#ifdef CPPHTTPLIB_SSL_ENABLED + // Create SSL client for HTTPS redirect + SSLClient redirect_client(host, port); + + // Setup basic client configuration first + setup_redirect_client(redirect_client); + + redirect_client.enable_server_certificate_verification( + server_certificate_verification_); + redirect_client.enable_server_hostname_verification( + server_hostname_verification_); + redirect_client.system_ca_mode_ = system_ca_mode_; + + // Transfer CA certificate to redirect client + if (!ca_cert_pem_.empty()) { + redirect_client.load_ca_cert_store(ca_cert_pem_.c_str(), + ca_cert_pem_.size()); + } + if (!ca_cert_file_path_.empty()) { + redirect_client.set_ca_cert_path(ca_cert_file_path_, ca_cert_dir_path_); + } + + // Client certificates are set through constructor for SSLClient + // NOTE: SSLClient constructor already takes client_cert_path and + // client_key_path so we need to create it properly if client certs are + // needed + + // Execute the redirect + return detail::redirect(redirect_client, req, res, path, location, error); +#else + // SSL not supported - set appropriate error + error = Error::SSLConnection; + output_error_log(error, &req); + return false; +#endif + } else { + // HTTP redirect + ClientImpl redirect_client(host, port); + + // Setup client with robust configuration + setup_redirect_client(redirect_client); + + // Execute the redirect + return detail::redirect(redirect_client, req, res, path, location, error); + } +} + +// New method for robust client setup (based on basic_manual_redirect.cpp +// logic) +template +inline void ClientImpl::setup_redirect_client(ClientType &client) { + // Copy basic settings first + client.set_connection_timeout(connection_timeout_sec_); + client.set_read_timeout(read_timeout_sec_, read_timeout_usec_); + client.set_write_timeout(write_timeout_sec_, write_timeout_usec_); + client.set_keep_alive(keep_alive_); + client.set_follow_location( + true); // Enable redirects to handle multi-step redirects + client.set_path_encode(path_encode_); + client.set_compress(compress_); + client.set_decompress(decompress_); + + // NOTE: Authentication credentials (basic auth, bearer token, digest auth) + // are intentionally NOT copied to the redirect client. Per RFC 9110 Section + // 15.4, credentials must not be forwarded when redirecting to a different + // host. This function is only called for cross-host redirects; same-host + // redirects are handled directly in ClientImpl::redirect(). + + // Copy the proxy configuration unconditionally; the per-target bypass is + // re-evaluated at send time, so a later hop to a non-bypassed host can + // still use the proxy. + client.no_proxy_entries_ = no_proxy_entries_; + if (!proxy_host_.empty() && proxy_port_ != -1) { + client.set_proxy(proxy_host_, proxy_port_); + + if (!proxy_basic_auth_username_.empty()) { + client.set_proxy_basic_auth(proxy_basic_auth_username_, + proxy_basic_auth_password_); + } + if (!proxy_bearer_token_auth_token_.empty()) { + client.set_proxy_bearer_token_auth(proxy_bearer_token_auth_token_); + } +#ifdef CPPHTTPLIB_SSL_ENABLED + if (!proxy_digest_auth_username_.empty()) { + client.set_proxy_digest_auth(proxy_digest_auth_username_, + proxy_digest_auth_password_); + } +#endif + } + + // Copy network and socket settings + client.set_address_family(address_family_); + client.set_tcp_nodelay(tcp_nodelay_); + client.set_ipv6_v6only(ipv6_v6only_); + if (socket_options_) { client.set_socket_options(socket_options_); } + if (!interface_.empty()) { client.set_interface(interface_); } + + // Copy logging and headers + if (logger_) { client.set_logger(logger_); } + if (error_logger_) { client.set_error_logger(error_logger_); } + + // NOTE: DO NOT copy default_headers_ as they may contain stale Host headers + // Each new client should generate its own headers based on its target host +} + +inline bool ClientImpl::write_content_with_provider(Stream &strm, + const Request &req, + Error &error) const { + auto is_shutting_down = []() { return false; }; + + if (req.is_chunked_content_provider_) { + auto compressor = compress_ ? detail::create_compressor().first + : std::unique_ptr(); + if (!compressor) { + compressor = detail::make_unique(); + } + + return detail::write_content_chunked(strm, req.content_provider_, + is_shutting_down, *compressor, error); + } else { + return detail::write_content_with_progress( + strm, req.content_provider_, 0, req.content_length_, is_shutting_down, + req.upload_progress, error); + } +} + +inline bool ClientImpl::write_request(Stream &strm, Request &req, + bool close_connection, Error &error, + bool skip_body) { + // Prepare additional headers + if (close_connection) { + if (!req.has_header("Connection")) { + req.set_header("Connection", "close"); + } + } + + std::string ct_for_defaults; + if (!req.has_header("Content-Type") && !req.body.empty()) { + ct_for_defaults = "text/plain"; + } + prepare_default_headers(req, false, ct_for_defaults); + + if (req.body.empty()) { + if (req.content_provider_) { + if (!req.is_chunked_content_provider_) { + if (!req.has_header("Content-Length")) { + auto length = std::to_string(req.content_length_); + req.set_header("Content-Length", length); + } + } + } else { + if (req.method == "POST" || req.method == "PUT" || + req.method == "PATCH") { + req.set_header("Content-Length", "0"); + } + } + } + + if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) { + if (!req.has_header("Authorization")) { + req.headers.insert(make_basic_authentication_header( + basic_auth_username_, basic_auth_password_, false)); + } + } + + if (!bearer_token_auth_token_.empty()) { + if (!req.has_header("Authorization")) { + req.headers.insert(make_bearer_token_authentication_header( + bearer_token_auth_token_, false)); + } + } + + // Proxy-Authorization is only sent when the proxy is actually used for + // this target — otherwise NO_PROXY-matched requests would leak proxy + // credentials directly to the destination server. + if (is_proxy_enabled_for_host(host_)) { + if (!proxy_basic_auth_username_.empty() && + !proxy_basic_auth_password_.empty() && + !req.has_header("Proxy-Authorization")) { + req.headers.insert(make_basic_authentication_header( + proxy_basic_auth_username_, proxy_basic_auth_password_, true)); + } + if (!proxy_bearer_token_auth_token_.empty() && + !req.has_header("Proxy-Authorization")) { + req.headers.insert(make_bearer_token_authentication_header( + proxy_bearer_token_auth_token_, true)); + } + } + + // Request line and headers + { + detail::BufferStream bstrm; + + // Extract path and query from req.path + std::string path_part, query_part; + auto query_pos = req.path.find('?'); + if (query_pos != std::string::npos) { + path_part = req.path.substr(0, query_pos); + query_part = req.path.substr(query_pos + 1); + } else { + path_part = req.path; + query_part = ""; + } + + // Encode path part. If the original `req.path` already contained a + // query component, preserve its raw query string (including parameter + // order) instead of reparsing and reassembling it which may reorder + // parameters due to container ordering (e.g. `Params` uses + // `std::multimap`). When there is no query in `req.path`, fall back to + // building a query from `req.params` so existing callers that pass + // `Params` continue to work. + auto path_with_query = + path_encode_ ? detail::encode_path(path_part) : path_part; + + if (!query_part.empty()) { + // Normalize the query string (decode then re-encode) while preserving + // the original parameter order. + auto normalized = detail::normalize_query_string(query_part); + if (!normalized.empty()) { path_with_query += '?' + normalized; } + + // Still populate req.params for handlers/users who read them. + detail::parse_query_text(query_part, req.params); + } else { + // No query in path; parse any query_part (empty) and append params + // from `req.params` when present (preserves prior behavior for + // callers who provide Params separately). + detail::parse_query_text(query_part, req.params); + if (!req.params.empty()) { + path_with_query = append_query_params(path_with_query, req.params); + } + } + + // Write request line and headers + detail::write_request_line(bstrm, req.method, path_with_query); + if (!detail::check_and_write_headers(bstrm, req.headers, header_writer_, + error)) { + output_error_log(error, &req); + return false; + } + + // Flush buffer + auto &data = bstrm.get_buffer(); + if (!detail::write_data(strm, data.data(), data.size())) { + error = Error::Write; + output_error_log(error, &req); + return false; + } + } + + // After sending request line and headers, wait briefly for an early server + // response (e.g. 4xx) and avoid sending a potentially large request body + // unnecessarily. This workaround is only enabled on Windows because Unix + // platforms surface write errors (EPIPE) earlier; on Windows kernel send + // buffering can accept large writes even when the peer already responded. + // Check the stream first (which covers SSL via `is_readable()`), then + // fall back to select on the socket. Only perform the wait for very large + // request bodies to avoid interfering with normal small requests and + // reduce side-effects. Poll briefly (up to 50ms as default) for an early + // response. Skip this check when using Expect: 100-continue, as the protocol + // handles early responses properly. +#if defined(_WIN32) + if (!skip_body && + req.body.size() > CPPHTTPLIB_WAIT_EARLY_SERVER_RESPONSE_THRESHOLD && + req.path.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) { + auto start = std::chrono::high_resolution_clock::now(); + + for (;;) { + // Prefer socket-level readiness to avoid SSL_pending() false-positives + // from SSL internals. If the underlying socket is readable, assume an + // early response may be present. + auto sock = strm.socket(); + if (sock != INVALID_SOCKET && detail::select_read(sock, 0, 0) > 0) { + return false; + } + + // Fallback to stream-level check for non-socket streams or when the + // socket isn't reporting readable. Avoid using `is_readable()` for + // SSL, since `SSL_pending()` may report buffered records that do not + // indicate a complete application-level response yet. + if (!is_ssl() && strm.is_readable()) { return false; } + + auto now = std::chrono::high_resolution_clock::now(); + auto elapsed = + std::chrono::duration_cast(now - start) + .count(); + if (elapsed >= CPPHTTPLIB_WAIT_EARLY_SERVER_RESPONSE_TIMEOUT_MSECOND) { + break; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } +#endif + + // Body + if (skip_body) { return true; } + + return write_request_body(strm, req, error); +} + +inline bool ClientImpl::write_request_body(Stream &strm, Request &req, + Error &error) { + if (req.body.empty()) { + return write_content_with_provider(strm, req, error); + } + + if (req.upload_progress) { + auto body_size = req.body.size(); + size_t written = 0; + auto data = req.body.data(); + + while (written < body_size) { + size_t to_write = (std::min)(CPPHTTPLIB_SEND_BUFSIZ, body_size - written); + if (!detail::write_data(strm, data + written, to_write)) { + error = Error::Write; + output_error_log(error, &req); + return false; + } + written += to_write; + + if (!req.upload_progress(written, body_size)) { + error = Error::Canceled; + output_error_log(error, &req); + return false; + } + } + } else { + if (!detail::write_data(strm, req.body.data(), req.body.size())) { + error = Error::Write; + output_error_log(error, &req); + return false; + } + } + + return true; +} + +inline std::unique_ptr +ClientImpl::send_with_content_provider_and_receiver( + Request &req, const char *body, size_t content_length, + ContentProvider content_provider, + ContentProviderWithoutLength content_provider_without_length, + const std::string &content_type, ContentReceiver content_receiver, + Error &error) { + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } + + auto enc = compress_ + ? detail::create_compressor() + : std::pair, const char *>( + nullptr, nullptr); + + if (enc.second) { req.set_header("Content-Encoding", enc.second); } + + if (enc.first && !content_provider_without_length) { + auto &compressor = enc.first; + + if (content_provider) { + auto ok = true; + size_t offset = 0; + DataSink data_sink; + + data_sink.write = [&](const char *data, size_t data_len) -> bool { + if (ok) { + auto last = offset + data_len == content_length; + + auto ret = compressor->compress( + data, data_len, last, + [&](const char *compressed_data, size_t compressed_data_len) { + req.body.append(compressed_data, compressed_data_len); + return true; + }); + + if (ret) { + offset += data_len; + } else { + ok = false; + } + } + return ok; + }; + + while (ok && offset < content_length) { + if (!content_provider(offset, content_length - offset, data_sink)) { + error = Error::Canceled; + output_error_log(error, &req); + return nullptr; + } + } + } else { + if (!compressor->compress(body, content_length, true, + [&](const char *data, size_t data_len) { + req.body.append(data, data_len); + return true; + })) { + error = Error::Compression; + output_error_log(error, &req); + return nullptr; + } + } + } else { + if (content_provider) { + req.content_length_ = content_length; + req.content_provider_ = std::move(content_provider); + req.is_chunked_content_provider_ = false; + } else if (content_provider_without_length) { + req.content_length_ = 0; + req.content_provider_ = detail::ContentProviderAdapter( + std::move(content_provider_without_length)); + req.is_chunked_content_provider_ = true; + req.set_header("Transfer-Encoding", "chunked"); + } else { + req.body.assign(body, content_length); + } + } + + if (content_receiver) { + req.content_receiver = + [content_receiver](const char *data, size_t data_length, + size_t /*offset*/, size_t /*total_length*/) { + return content_receiver(data, data_length); + }; + } + + auto res = detail::make_unique(); + return send(req, *res, error) ? std::move(res) : nullptr; +} + +inline Result ClientImpl::send_with_content_provider_and_receiver( + const std::string &method, const std::string &path, const Headers &headers, + const char *body, size_t content_length, ContentProvider content_provider, + ContentProviderWithoutLength content_provider_without_length, + const std::string &content_type, ContentReceiver content_receiver, + UploadProgress progress) { + Request req; + req.method = method; + req.headers = headers; + req.path = path; + req.upload_progress = std::move(progress); + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + auto error = Error::Success; + + auto res = send_with_content_provider_and_receiver( + req, body, content_length, std::move(content_provider), + std::move(content_provider_without_length), content_type, + std::move(content_receiver), error); + +#ifdef CPPHTTPLIB_SSL_ENABLED + return Result{std::move(res), error, std::move(req.headers), last_ssl_error_, + last_backend_error_}; +#else + return Result{std::move(res), error, std::move(req.headers)}; +#endif +} + +inline void ClientImpl::output_log(const Request &req, + const Response &res) const { + if (logger_) { + std::lock_guard guard(logger_mutex_); + logger_(req, res); + } +} + +inline void ClientImpl::output_error_log(const Error &err, + const Request *req) const { + if (error_logger_) { + std::lock_guard guard(logger_mutex_); + error_logger_(err, req); + } +} + +inline bool ClientImpl::process_request(Stream &strm, Request &req, + Response &res, bool close_connection, + Error &error) { + // Auto-add Expect: 100-continue for large bodies + if (CPPHTTPLIB_EXPECT_100_THRESHOLD > 0 && !req.has_header("Expect")) { + auto body_size = req.body.empty() ? req.content_length_ : req.body.size(); + if (body_size >= CPPHTTPLIB_EXPECT_100_THRESHOLD) { + req.set_header("Expect", "100-continue"); + } + } + + // Check for Expect: 100-continue + auto expect_100_continue = req.get_header_value("Expect") == "100-continue"; + + // Send request (skip body if using Expect: 100-continue) + auto write_request_success = + write_request(strm, req, close_connection, error, expect_100_continue); + +#ifdef CPPHTTPLIB_SSL_ENABLED + if (is_ssl() && !expect_100_continue) { + auto is_proxy_enabled = is_proxy_enabled_for_host(host_); + if (!is_proxy_enabled) { + if (tls::is_peer_closed(socket_.ssl, socket_.sock)) { + error = Error::SSLPeerCouldBeClosed_; + output_error_log(error, &req); + return false; + } + } + } +#endif + + // Handle Expect: 100-continue. + // + // Wait for an interim/early response by attempting to read the status line + // under a short timeout, instead of trusting raw socket readability. Over + // TLS, post-handshake records (e.g. session tickets) make the socket + // readable without any HTTP response being available; relying on + // `select_read` there caused the body to be withheld forever and the + // request to fail with `Read` (#2458). If no status line arrives within the + // timeout, send the body anyway (matching curl's behavior). + auto status_line_read = false; + if (expect_100_continue && write_request_success) { + if (CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND > 0) { + time_t sec = CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND / 1000; + time_t usec = (CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND % 1000) * 1000; + strm.set_read_timeout(sec, usec); + status_line_read = read_response_line(strm, req, res, false); + strm.set_read_timeout(read_timeout_sec_, read_timeout_usec_); + } + + if (!status_line_read) { + // No interim response within the timeout: send the body and handle the + // response as usual. + if (!write_request_body(strm, req, error)) { return false; } + expect_100_continue = false; // Switch to normal response handling + } + } + + // Receive response and headers + // When using Expect: 100-continue, don't auto-skip `100 Continue` response + if ((!status_line_read && + !read_response_line(strm, req, res, !expect_100_continue)) || + !detail::read_headers(strm, res.headers)) { + if (write_request_success) { error = Error::Read; } + output_error_log(error, &req); + return false; + } + + if (!write_request_success) { return false; } + + // Handle Expect: 100-continue response + if (expect_100_continue) { + if (res.status == StatusCode::Continue_100) { + // Server accepted, send the body + if (!write_request_body(strm, req, error)) { return false; } + + // Read the actual response + res.headers.clear(); + res.body.clear(); + if (!read_response_line(strm, req, res) || + !detail::read_headers(strm, res.headers)) { + error = Error::Read; + output_error_log(error, &req); + return false; + } + } + // If not 100 Continue, server returned an error; proceed with that response + } + + // Body + if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" && + req.method != "CONNECT") { + auto redirect = 300 < res.status && res.status < 400 && + res.status != StatusCode::NotModified_304 && + follow_location_; + + if (req.response_handler && !redirect) { + if (!req.response_handler(res)) { + error = Error::Canceled; + output_error_log(error, &req); + return false; + } + } + + auto out = + req.content_receiver + ? static_cast( + [&](const char *buf, size_t n, size_t off, size_t len) { + if (redirect) { return true; } + auto ret = req.content_receiver(buf, n, off, len); + if (!ret) { + error = Error::Canceled; + output_error_log(error, &req); + } + return ret; + }) + : static_cast( + [&](const char *buf, size_t n, size_t /*off*/, + size_t /*len*/) { + assert(res.body.size() + n <= res.body.max_size()); + if (payload_max_length_ > 0 && + (res.body.size() >= payload_max_length_ || + n > payload_max_length_ - res.body.size())) { + return false; + } + res.body.append(buf, n); + return true; + }); + + auto progress = [&](size_t current, size_t total) { + if (!req.download_progress || redirect) { return true; } + auto ret = req.download_progress(current, total); + if (!ret) { + error = Error::Canceled; + output_error_log(error, &req); + } + return ret; + }; + + if (res.has_header("Content-Length")) { + if (!req.content_receiver) { + auto len = res.get_header_value_u64("Content-Length"); + if (len > res.body.max_size()) { + error = Error::Read; + output_error_log(error, &req); + return false; + } + // Cap the reservation by payload_max_length_ to avoid OOM when a + // hostile or malformed server sends an enormous Content-Length. + // The actual body read below is bounded by payload_max_length_, + // so reserving more than that is never useful. + auto reserve_len = static_cast(len); + if (payload_max_length_ > 0 && reserve_len > payload_max_length_) { + reserve_len = payload_max_length_; + } + res.body.reserve(reserve_len); + } + } + + if (res.status != StatusCode::NotModified_304) { + int dummy_status; + auto max_length = (!has_payload_max_length_ && req.content_receiver) + ? (std::numeric_limits::max)() + : payload_max_length_; + if (!detail::read_content(strm, res, max_length, dummy_status, + std::move(progress), std::move(out), + decompress_)) { + if (error != Error::Canceled) { error = Error::Read; } + output_error_log(error, &req); + return false; + } + } + } + + // Log + output_log(req, res); + + return true; +} + +inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider( + const std::string &boundary, const UploadFormDataItems &items, + const FormDataProviderItems &provider_items) const { + size_t cur_item = 0; + size_t cur_start = 0; + // cur_item and cur_start are copied to within the std::function and + // maintain state between successive calls + return [&, cur_item, cur_start](size_t offset, + DataSink &sink) mutable -> bool { + if (!offset && !items.empty()) { + sink.os << detail::serialize_multipart_formdata(items, boundary, false); + return true; + } else if (cur_item < provider_items.size()) { + if (!cur_start) { + const auto &begin = detail::serialize_multipart_formdata_item_begin( + provider_items[cur_item], boundary); + offset += begin.size(); + cur_start = offset; + sink.os << begin; + } + + DataSink cur_sink; + auto has_data = true; + cur_sink.write = sink.write; + cur_sink.done = [&]() { has_data = false; }; + + if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) { + return false; + } + + if (!has_data) { + sink.os << detail::serialize_multipart_formdata_item_end(); + cur_item++; + cur_start = 0; + } + return true; + } else { + sink.os << detail::serialize_multipart_formdata_finish(boundary); + sink.done(); + return true; + } + }; +} + +inline bool ClientImpl::process_socket( + const Socket &socket, + std::chrono::time_point start_time, + std::function callback) { + return detail::process_client_socket( + socket.sock, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, + write_timeout_usec_, max_timeout_msec_, start_time, std::move(callback)); +} + +inline bool ClientImpl::is_ssl() const { return false; } + +inline Result ClientImpl::Get(const std::string &path, + DownloadProgress progress) { + return Get(path, Headers(), std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, const Params ¶ms, + const Headers &headers, + DownloadProgress progress) { + if (params.empty()) { return Get(path, headers); } + + std::string path_with_query = append_query_params(path, params); + return Get(path_with_query, headers, std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, const Headers &headers, + DownloadProgress progress) { + Request req; + req.method = "GET"; + req.path = path; + req.headers = headers; + req.download_progress = std::move(progress); + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Get(const std::string &path, + ContentReceiver content_receiver, + DownloadProgress progress) { + return Get(path, Headers(), nullptr, std::move(content_receiver), + std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, const Headers &headers, + ContentReceiver content_receiver, + DownloadProgress progress) { + return Get(path, headers, nullptr, std::move(content_receiver), + std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + return Get(path, Headers(), std::move(response_handler), + std::move(content_receiver), std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + Request req; + req.method = "GET"; + req.path = path; + req.headers = headers; + req.response_handler = std::move(response_handler); + req.content_receiver = + [content_receiver](const char *data, size_t data_length, + size_t /*offset*/, size_t /*total_length*/) { + return content_receiver(data, data_length); + }; + req.download_progress = std::move(progress); + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Get(const std::string &path, const Params ¶ms, + const Headers &headers, + ContentReceiver content_receiver, + DownloadProgress progress) { + return Get(path, params, headers, nullptr, std::move(content_receiver), + std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, const Params ¶ms, + const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + if (params.empty()) { + return Get(path, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); + } + + std::string path_with_query = append_query_params(path, params); + return Get(path_with_query, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); +} + +inline Result ClientImpl::Head(const std::string &path) { + return Head(path, Headers()); +} + +inline Result ClientImpl::Head(const std::string &path, + const Headers &headers) { + Request req; + req.method = "HEAD"; + req.headers = headers; + req.path = path; + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Post(const std::string &path) { + return Post(path, std::string(), std::string()); +} + +inline Result ClientImpl::Post(const std::string &path, + const Headers &headers) { + return Post(path, headers, nullptr, 0, std::string()); +} + +inline Result ClientImpl::Post(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return Post(path, Headers(), body, content_length, content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return Post(path, Headers(), body, content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Params ¶ms) { + return Post(path, Headers(), params); +} + +inline Result ClientImpl::Post(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return Post(path, Headers(), content_length, std::move(content_provider), + content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return Post(path, Headers(), content_length, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} + +inline Result ClientImpl::Post(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return Post(path, Headers(), std::move(content_provider), content_type, + progress); +} + +inline Result ClientImpl::Post(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return Post(path, Headers(), std::move(content_provider), content_type, + std::move(content_receiver), progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const Params ¶ms) { + auto query = detail::params_to_query_str(params); + return Post(path, headers, query, "application/x-www-form-urlencoded"); +} + +inline Result ClientImpl::Post(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return Post(path, Headers(), items, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Post(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + if (!detail::is_multipart_boundary_chars_valid(boundary)) { + return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; + } + + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Post(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "POST", path, headers, body, content_length, nullptr, nullptr, + content_type, nullptr, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "POST", path, headers, body.data(), body.size(), nullptr, nullptr, + content_type, nullptr, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "POST", path, headers, nullptr, content_length, + std::move(content_provider), nullptr, content_type, nullptr, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + return send_with_content_provider_and_receiver( + "POST", path, headers, nullptr, content_length, + std::move(content_provider), nullptr, content_type, + std::move(content_receiver), std::move(progress)); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "POST", path, headers, nullptr, 0, nullptr, std::move(content_provider), + content_type, nullptr, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + return send_with_content_provider_and_receiver( + "POST", path, headers, nullptr, 0, nullptr, std::move(content_provider), + content_type, std::move(content_receiver), std::move(progress)); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider_and_receiver( + "POST", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type, nullptr, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + Request req; + req.method = "POST"; + req.path = path; + req.headers = headers; + req.body = body; + req.content_receiver = + [content_receiver](const char *data, size_t data_length, + size_t /*offset*/, size_t /*total_length*/) { + return content_receiver(data, data_length); + }; + req.download_progress = std::move(progress); + + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Put(const std::string &path) { + return Put(path, std::string(), std::string()); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers) { + return Put(path, headers, nullptr, 0, std::string()); +} + +inline Result ClientImpl::Put(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return Put(path, Headers(), body, content_length, content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return Put(path, Headers(), body, content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Params ¶ms) { + return Put(path, Headers(), params); +} + +inline Result ClientImpl::Put(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return Put(path, Headers(), content_length, std::move(content_provider), + content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return Put(path, Headers(), content_length, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} + +inline Result ClientImpl::Put(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return Put(path, Headers(), std::move(content_provider), content_type, + progress); +} + +inline Result ClientImpl::Put(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return Put(path, Headers(), std::move(content_provider), content_type, + std::move(content_receiver), progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const Params ¶ms) { + auto query = detail::params_to_query_str(params); + return Put(path, headers, query, "application/x-www-form-urlencoded"); +} + +inline Result ClientImpl::Put(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return Put(path, Headers(), items, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Put(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + if (!detail::is_multipart_boundary_chars_valid(boundary)) { + return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; + } + + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Put(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PUT", path, headers, body, content_length, nullptr, nullptr, + content_type, nullptr, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PUT", path, headers, body.data(), body.size(), nullptr, nullptr, + content_type, nullptr, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PUT", path, headers, nullptr, content_length, + std::move(content_provider), nullptr, content_type, nullptr, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PUT", path, headers, nullptr, content_length, + std::move(content_provider), nullptr, content_type, + std::move(content_receiver), progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PUT", path, headers, nullptr, 0, nullptr, std::move(content_provider), + content_type, nullptr, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PUT", path, headers, nullptr, 0, nullptr, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider_and_receiver( + "PUT", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type, nullptr, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + Request req; + req.method = "PUT"; + req.path = path; + req.headers = headers; + req.body = body; + req.content_receiver = + [content_receiver](const char *data, size_t data_length, + size_t /*offset*/, size_t /*total_length*/) { + return content_receiver(data, data_length); + }; + req.download_progress = std::move(progress); + + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Patch(const std::string &path) { + return Patch(path, std::string(), std::string()); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + UploadProgress progress) { + return Patch(path, headers, nullptr, 0, std::string(), progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return Patch(path, Headers(), body, content_length, content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return Patch(path, Headers(), body, content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Params ¶ms) { + return Patch(path, Headers(), params); +} + +inline Result ClientImpl::Patch(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return Patch(path, Headers(), content_length, std::move(content_provider), + content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return Patch(path, Headers(), content_length, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} + +inline Result ClientImpl::Patch(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return Patch(path, Headers(), std::move(content_provider), content_type, + progress); +} + +inline Result ClientImpl::Patch(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return Patch(path, Headers(), std::move(content_provider), content_type, + std::move(content_receiver), progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const Params ¶ms) { + auto query = detail::params_to_query_str(params); + return Patch(path, headers, query, "application/x-www-form-urlencoded"); +} + +inline Result ClientImpl::Patch(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return Patch(path, Headers(), items, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Patch(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + if (!detail::is_multipart_boundary_chars_valid(boundary)) { + return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; + } + + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Patch(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PATCH", path, headers, body, content_length, nullptr, nullptr, + content_type, nullptr, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PATCH", path, headers, body.data(), body.size(), nullptr, nullptr, + content_type, nullptr, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PATCH", path, headers, nullptr, content_length, + std::move(content_provider), nullptr, content_type, nullptr, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PATCH", path, headers, nullptr, content_length, + std::move(content_provider), nullptr, content_type, + std::move(content_receiver), progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PATCH", path, headers, nullptr, 0, nullptr, std::move(content_provider), + content_type, nullptr, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return send_with_content_provider_and_receiver( + "PATCH", path, headers, nullptr, 0, nullptr, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider_and_receiver( + "PATCH", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type, nullptr, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + Request req; + req.method = "PATCH"; + req.path = path; + req.headers = headers; + req.body = body; + req.content_receiver = + [content_receiver](const char *data, size_t data_length, + size_t /*offset*/, size_t /*total_length*/) { + return content_receiver(data, data_length); + }; + req.download_progress = std::move(progress); + + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Delete(const std::string &path, + DownloadProgress progress) { + return Delete(path, Headers(), std::string(), std::string(), progress); +} + +inline Result ClientImpl::Delete(const std::string &path, + const Headers &headers, + DownloadProgress progress) { + return Delete(path, headers, std::string(), std::string(), progress); +} + +inline Result ClientImpl::Delete(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + DownloadProgress progress) { + return Delete(path, Headers(), body, content_length, content_type, progress); +} + +inline Result ClientImpl::Delete(const std::string &path, + const std::string &body, + const std::string &content_type, + DownloadProgress progress) { + return Delete(path, Headers(), body.data(), body.size(), content_type, + progress); +} + +inline Result ClientImpl::Delete(const std::string &path, + const Headers &headers, + const std::string &body, + const std::string &content_type, + DownloadProgress progress) { + return Delete(path, headers, body.data(), body.size(), content_type, + progress); +} + +inline Result ClientImpl::Delete(const std::string &path, const Params ¶ms, + DownloadProgress progress) { + return Delete(path, Headers(), params, progress); +} + +inline Result ClientImpl::Delete(const std::string &path, + const Headers &headers, const Params ¶ms, + DownloadProgress progress) { + auto query = detail::params_to_query_str(params); + return Delete(path, headers, query, "application/x-www-form-urlencoded", + progress); +} + +inline Result ClientImpl::Delete(const std::string &path, + const Headers &headers, const char *body, + size_t content_length, + const std::string &content_type, + DownloadProgress progress) { + Request req; + req.method = "DELETE"; + req.headers = headers; + req.path = path; + req.download_progress = std::move(progress); + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } + req.body.assign(body, content_length); + + return send_(std::move(req)); +} + +inline Result ClientImpl::Options(const std::string &path) { + return Options(path, Headers()); +} + +inline Result ClientImpl::Options(const std::string &path, + const Headers &headers) { + Request req; + req.method = "OPTIONS"; + req.headers = headers; + req.path = path; + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + return send_(std::move(req)); +} + +inline void ClientImpl::stop() { + std::lock_guard guard(socket_mutex_); + + // If there is anything ongoing right now, the ONLY thread-safe thing we can + // do is to shutdown_socket, so that threads using this socket suddenly + // discover they can't read/write any more and error out. Everything else + // (closing the socket, shutting ssl down) is unsafe because these actions + // are not thread-safe. + if (socket_requests_in_flight_ > 0) { + shutdown_socket(socket_); + + // Aside from that, we set a flag for the socket to be closed when we're + // done. + socket_should_be_closed_when_request_is_done_ = true; + return; + } + + disconnect(/*gracefully=*/true); +} + +inline std::string ClientImpl::host() const { return host_; } + +inline int ClientImpl::port() const { return port_; } + +inline size_t ClientImpl::is_socket_open() const { + std::lock_guard guard(socket_mutex_); + return socket_.is_open(); +} + +inline socket_t ClientImpl::socket() const { return socket_.sock; } + +inline void ClientImpl::set_connection_timeout(time_t sec, time_t usec) { + connection_timeout_sec_ = sec; + connection_timeout_usec_ = usec; +} + +inline void ClientImpl::set_read_timeout(time_t sec, time_t usec) { + read_timeout_sec_ = sec; + read_timeout_usec_ = usec; +} + +inline void ClientImpl::set_write_timeout(time_t sec, time_t usec) { + write_timeout_sec_ = sec; + write_timeout_usec_ = usec; +} + +inline void ClientImpl::set_max_timeout(time_t msec) { + max_timeout_msec_ = msec; +} + +inline void ClientImpl::set_basic_auth(const std::string &username, + const std::string &password) { + basic_auth_username_ = username; + basic_auth_password_ = password; +} + +inline void ClientImpl::set_bearer_token_auth(const std::string &token) { + bearer_token_auth_token_ = token; +} + +inline void ClientImpl::set_keep_alive(bool on) { keep_alive_ = on; } + +inline void ClientImpl::set_follow_location(bool on) { follow_location_ = on; } + +inline void ClientImpl::set_path_encode(bool on) { path_encode_ = on; } + +inline void +ClientImpl::set_hostname_addr_map(std::map addr_map) { + addr_map_ = std::move(addr_map); +} + +inline void ClientImpl::set_default_headers(Headers headers) { + default_headers_ = std::move(headers); +} + +inline void ClientImpl::set_header_writer( + std::function const &writer) { + header_writer_ = writer; +} + +inline void ClientImpl::set_address_family(int family) { + address_family_ = family; +} + +inline void ClientImpl::set_tcp_nodelay(bool on) { tcp_nodelay_ = on; } + +inline void ClientImpl::set_ipv6_v6only(bool on) { ipv6_v6only_ = on; } + +inline void ClientImpl::set_socket_options(SocketOptions socket_options) { + socket_options_ = std::move(socket_options); +} + +inline void ClientImpl::set_compress(bool on) { compress_ = on; } + +inline void ClientImpl::set_decompress(bool on) { decompress_ = on; } + +inline void ClientImpl::set_payload_max_length(size_t length) { + payload_max_length_ = length; + has_payload_max_length_ = true; +} + +inline void ClientImpl::set_interface(const std::string &intf) { + interface_ = intf; +} + +inline void ClientImpl::set_proxy(const std::string &host, int port) { + proxy_host_ = host; + proxy_port_ = port; + std::lock_guard guard(socket_mutex_); + disconnect(/*gracefully=*/true); +} + +inline void ClientImpl::set_proxy_basic_auth(const std::string &username, + const std::string &password) { + proxy_basic_auth_username_ = username; + proxy_basic_auth_password_ = password; +} + +inline void ClientImpl::set_proxy_bearer_token_auth(const std::string &token) { + proxy_bearer_token_auth_token_ = token; +} + +inline void ClientImpl::set_no_proxy(const std::vector &patterns) { + std::vector parsed; + parsed.reserve(patterns.size()); + for (const auto &p : patterns) { + auto trimmed = detail::trim_copy(p); + if (trimmed.empty()) { continue; } + detail::NoProxyEntry entry; + if (detail::parse_no_proxy_entry(trimmed, entry)) { + parsed.push_back(std::move(entry)); + } + } + no_proxy_entries_ = std::move(parsed); + std::lock_guard guard(socket_mutex_); + disconnect(/*gracefully=*/true); +} + +#ifdef CPPHTTPLIB_SSL_ENABLED +inline void ClientImpl::set_digest_auth(const std::string &username, + const std::string &password) { + digest_auth_username_ = username; + digest_auth_password_ = password; +} + +inline void ClientImpl::set_ca_cert_path(const std::string &ca_cert_file_path, + const std::string &ca_cert_dir_path) { + ca_cert_file_path_ = ca_cert_file_path; + ca_cert_dir_path_ = ca_cert_dir_path; +} + +inline void ClientImpl::set_proxy_digest_auth(const std::string &username, + const std::string &password) { + proxy_digest_auth_username_ = username; + proxy_digest_auth_password_ = password; +} + +inline void ClientImpl::enable_server_certificate_verification(bool enabled) { + server_certificate_verification_ = enabled; +} + +inline void ClientImpl::enable_server_hostname_verification(bool enabled) { + server_hostname_verification_ = enabled; +} + +inline void ClientImpl::enable_system_ca(bool enabled) { + system_ca_mode_ = enabled ? SystemCAMode::Enabled : SystemCAMode::Disabled; +} +#endif + +inline void ClientImpl::set_logger(Logger logger) { + logger_ = std::move(logger); +} + +inline void ClientImpl::set_error_logger(ErrorLogger error_logger) { + error_logger_ = std::move(error_logger); +} + +/* + * SSL/TLS Common Implementation + */ + +inline ClientConnection::~ClientConnection() { +#ifdef CPPHTTPLIB_SSL_ENABLED + if (session) { + tls::shutdown(session, true); + tls::free_session(session); + session = nullptr; + } +#endif + + if (sock != INVALID_SOCKET) { + detail::close_socket(sock); + sock = INVALID_SOCKET; + } +} + +// Universal client implementation +inline Client::Client(const std::string &scheme_host_port) + : Client(scheme_host_port, std::string(), std::string()) {} + +inline Client::Client(const std::string &scheme_host_port, + const std::string &client_cert_path, + const std::string &client_key_path) { + detail::UrlComponents uc; + if (detail::parse_url(scheme_host_port, uc) && !uc.host.empty()) { + auto &scheme = uc.scheme; + +#ifdef CPPHTTPLIB_SSL_ENABLED + if (!scheme.empty() && (scheme != "http" && scheme != "https")) { +#else + if (!scheme.empty() && scheme != "http") { +#endif +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + std::string msg = "'" + scheme + "' scheme is not supported."; + throw std::invalid_argument(msg); +#endif + return; + } + + auto is_ssl = scheme == "https"; + + auto host = std::move(uc.host); + + auto port = is_ssl ? 443 : 80; + if (!uc.port.empty() && !detail::parse_port(uc.port, port)) { return; } + + if (is_ssl) { +#ifdef CPPHTTPLIB_SSL_ENABLED + cli_ = detail::make_unique(host, port, client_cert_path, + client_key_path); + is_ssl_ = is_ssl; +#endif + } else { + cli_ = detail::make_unique(host, port, client_cert_path, + client_key_path); + } + } else { + // NOTE: Update TEST(UniversalClientImplTest, Ipv6LiteralAddress) + // if port param below changes. + cli_ = detail::make_unique(scheme_host_port, 80, + client_cert_path, client_key_path); + } +} + +inline Client::Client(const std::string &host, int port) + : Client(host, port, std::string(), std::string()) {} + +inline Client::Client(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path) + : cli_(detail::make_unique(host, port, client_cert_path, + client_key_path)) {} + +inline Client::~Client() = default; + +inline bool Client::is_valid() const { + return cli_ != nullptr && cli_->is_valid(); +} + +inline Result Client::Get(const std::string &path, DownloadProgress progress) { + return cli_->Get(path, std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Headers &headers, + DownloadProgress progress) { + return cli_->Get(path, headers, std::move(progress)); +} +inline Result Client::Get(const std::string &path, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, std::move(content_receiver), std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Headers &headers, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, headers, std::move(content_receiver), + std::move(progress)); +} +inline Result Client::Get(const std::string &path, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, std::move(response_handler), + std::move(content_receiver), std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Params ¶ms, + const Headers &headers, DownloadProgress progress) { + return cli_->Get(path, params, headers, std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Params ¶ms, + const Headers &headers, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, params, headers, std::move(content_receiver), + std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Params ¶ms, + const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, params, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); +} + +inline Result Client::Head(const std::string &path) { return cli_->Head(path); } +inline Result Client::Head(const std::string &path, const Headers &headers) { + return cli_->Head(path, headers); +} + +inline Result Client::Post(const std::string &path) { return cli_->Post(path); } +inline Result Client::Post(const std::string &path, const Headers &headers) { + return cli_->Post(path, headers); +} +inline Result Client::Post(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, body, content_length, content_type, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, headers, body, content_length, content_type, + progress); +} +inline Result Client::Post(const std::string &path, const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, body, content_type, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, headers, body, content_type, progress); +} +inline Result Client::Post(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Post(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return cli_->Post(path, content_length, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} +inline Result Client::Post(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, std::move(content_provider), content_type, progress); +} +inline Result Client::Post(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return cli_->Post(path, std::move(content_provider), content_type, + std::move(content_receiver), progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, headers, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Post(path, headers, content_length, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, headers, std::move(content_provider), content_type, + progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Post(path, headers, std::move(content_provider), content_type, + std::move(content_receiver), progress); +} +inline Result Client::Post(const std::string &path, const Params ¶ms) { + return cli_->Post(path, params); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const Params ¶ms) { + return cli_->Post(path, headers, params); +} +inline Result Client::Post(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Post(path, items, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Post(path, headers, items, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + return cli_->Post(path, headers, items, boundary, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + return cli_->Post(path, headers, items, provider_items, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Post(path, headers, body, content_type, + std::move(content_receiver), progress); +} + +inline Result Client::Put(const std::string &path) { return cli_->Put(path); } +inline Result Client::Put(const std::string &path, const Headers &headers) { + return cli_->Put(path, headers); +} +inline Result Client::Put(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, body, content_length, content_type, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, headers, body, content_length, content_type, progress); +} +inline Result Client::Put(const std::string &path, const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, body, content_type, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, headers, body, content_type, progress); +} +inline Result Client::Put(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Put(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return cli_->Put(path, content_length, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} +inline Result Client::Put(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, std::move(content_provider), content_type, progress); +} +inline Result Client::Put(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return cli_->Put(path, std::move(content_provider), content_type, + std::move(content_receiver), progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, headers, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return cli_->Put(path, headers, content_length, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, headers, std::move(content_provider), content_type, + progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return cli_->Put(path, headers, std::move(content_provider), content_type, + std::move(content_receiver), progress); +} +inline Result Client::Put(const std::string &path, const Params ¶ms) { + return cli_->Put(path, params); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const Params ¶ms) { + return cli_->Put(path, headers, params); +} +inline Result Client::Put(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Put(path, items, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Put(path, headers, items, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + return cli_->Put(path, headers, items, boundary, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + return cli_->Put(path, headers, items, provider_items, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Put(path, headers, body, content_type, content_receiver, + progress); +} + +inline Result Client::Patch(const std::string &path) { + return cli_->Patch(path); +} +inline Result Client::Patch(const std::string &path, const Headers &headers) { + return cli_->Patch(path, headers); +} +inline Result Client::Patch(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, body, content_length, content_type, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, headers, body, content_length, content_type, + progress); +} +inline Result Client::Patch(const std::string &path, const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, body, content_type, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, headers, body, content_type, progress); +} +inline Result Client::Patch(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Patch(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return cli_->Patch(path, content_length, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} +inline Result Client::Patch(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, std::move(content_provider), content_type, progress); +} +inline Result Client::Patch(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return cli_->Patch(path, std::move(content_provider), content_type, + std::move(content_receiver), progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, headers, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return cli_->Patch(path, headers, content_length, std::move(content_provider), + content_type, std::move(content_receiver), progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, headers, std::move(content_provider), content_type, + progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + ContentReceiver content_receiver, + UploadProgress progress) { + return cli_->Patch(path, headers, std::move(content_provider), content_type, + std::move(content_receiver), progress); +} +inline Result Client::Patch(const std::string &path, const Params ¶ms) { + return cli_->Patch(path, params); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const Params ¶ms) { + return cli_->Patch(path, headers, params); +} +inline Result Client::Patch(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Patch(path, items, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Patch(path, headers, items, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + return cli_->Patch(path, headers, items, boundary, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + return cli_->Patch(path, headers, items, provider_items, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Patch(path, headers, body, content_type, content_receiver, + progress); +} + +inline Result Client::Delete(const std::string &path, + DownloadProgress progress) { + return cli_->Delete(path, progress); +} +inline Result Client::Delete(const std::string &path, const Headers &headers, + DownloadProgress progress) { + return cli_->Delete(path, headers, progress); +} +inline Result Client::Delete(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + DownloadProgress progress) { + return cli_->Delete(path, body, content_length, content_type, progress); +} +inline Result Client::Delete(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + DownloadProgress progress) { + return cli_->Delete(path, headers, body, content_length, content_type, + progress); +} +inline Result Client::Delete(const std::string &path, const std::string &body, + const std::string &content_type, + DownloadProgress progress) { + return cli_->Delete(path, body, content_type, progress); +} +inline Result Client::Delete(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + DownloadProgress progress) { + return cli_->Delete(path, headers, body, content_type, progress); +} +inline Result Client::Delete(const std::string &path, const Params ¶ms, + DownloadProgress progress) { + return cli_->Delete(path, params, progress); +} +inline Result Client::Delete(const std::string &path, const Headers &headers, + const Params ¶ms, DownloadProgress progress) { + return cli_->Delete(path, headers, params, progress); +} + +inline Result Client::Options(const std::string &path) { + return cli_->Options(path); +} +inline Result Client::Options(const std::string &path, const Headers &headers) { + return cli_->Options(path, headers); +} + +inline ClientImpl::StreamHandle +Client::open_stream(const std::string &method, const std::string &path, + const Params ¶ms, const Headers &headers, + const std::string &body, const std::string &content_type) { + return cli_->open_stream(method, path, params, headers, body, content_type); +} + +inline bool Client::send(Request &req, Response &res, Error &error) { + return cli_->send(req, res, error); +} + +inline Result Client::send(const Request &req) { return cli_->send(req); } + +inline void Client::stop() { cli_->stop(); } + +inline std::string Client::host() const { return cli_->host(); } + +inline int Client::port() const { return cli_->port(); } + +inline size_t Client::is_socket_open() const { return cli_->is_socket_open(); } + +inline socket_t Client::socket() const { return cli_->socket(); } + +inline void +Client::set_hostname_addr_map(std::map addr_map) { + cli_->set_hostname_addr_map(std::move(addr_map)); +} + +inline void Client::set_default_headers(Headers headers) { + cli_->set_default_headers(std::move(headers)); +} + +inline void Client::set_header_writer( + std::function const &writer) { + cli_->set_header_writer(writer); +} + +inline void Client::set_address_family(int family) { + cli_->set_address_family(family); +} + +inline void Client::set_tcp_nodelay(bool on) { cli_->set_tcp_nodelay(on); } + +inline void Client::set_socket_options(SocketOptions socket_options) { + cli_->set_socket_options(std::move(socket_options)); +} + +inline void Client::set_connection_timeout(time_t sec, time_t usec) { + cli_->set_connection_timeout(sec, usec); +} + +inline void Client::set_read_timeout(time_t sec, time_t usec) { + cli_->set_read_timeout(sec, usec); +} + +inline void Client::set_write_timeout(time_t sec, time_t usec) { + cli_->set_write_timeout(sec, usec); +} + +inline void Client::set_basic_auth(const std::string &username, + const std::string &password) { + cli_->set_basic_auth(username, password); +} +inline void Client::set_bearer_token_auth(const std::string &token) { + cli_->set_bearer_token_auth(token); +} + +inline void Client::set_keep_alive(bool on) { cli_->set_keep_alive(on); } +inline void Client::set_follow_location(bool on) { + cli_->set_follow_location(on); +} + +inline void Client::set_path_encode(bool on) { cli_->set_path_encode(on); } + +inline void Client::set_compress(bool on) { cli_->set_compress(on); } + +inline void Client::set_decompress(bool on) { cli_->set_decompress(on); } + +inline void Client::set_payload_max_length(size_t length) { + cli_->set_payload_max_length(length); +} + +inline void Client::set_interface(const std::string &intf) { + cli_->set_interface(intf); +} + +inline void Client::set_proxy(const std::string &host, int port) { + cli_->set_proxy(host, port); +} +inline void Client::set_proxy_basic_auth(const std::string &username, + const std::string &password) { + cli_->set_proxy_basic_auth(username, password); +} +inline void Client::set_proxy_bearer_token_auth(const std::string &token) { + cli_->set_proxy_bearer_token_auth(token); +} +inline void Client::set_no_proxy(const std::vector &patterns) { + cli_->set_no_proxy(patterns); +} + +inline void Client::set_logger(Logger logger) { + cli_->set_logger(std::move(logger)); +} + +inline void Client::set_error_logger(ErrorLogger error_logger) { + cli_->set_error_logger(std::move(error_logger)); +} + +/* + * Group 6: SSL Server and Client implementation + */ + +#ifdef CPPHTTPLIB_SSL_ENABLED + +// SSL HTTP server implementation +inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path, + const char *client_ca_cert_file_path, + const char *client_ca_cert_dir_path, + const char *private_key_password) { + using namespace tls; + + ctx_ = create_server_context(); + if (!ctx_) { return; } + + // Load server certificate and private key + if (!set_server_cert_file(ctx_, cert_path, private_key_path, + private_key_password)) { + last_ssl_error_ = static_cast(get_error()); + free_context(ctx_); + ctx_ = nullptr; + return; + } + + // Load client CA certificates for client authentication + if (client_ca_cert_file_path || client_ca_cert_dir_path) { + if (!set_client_ca_file(ctx_, client_ca_cert_file_path, + client_ca_cert_dir_path)) { + last_ssl_error_ = static_cast(get_error()); + free_context(ctx_); + ctx_ = nullptr; + return; + } + // Enable client certificate verification + set_verify_client(ctx_, true); + } +} + +inline SSLServer::SSLServer(const PemMemory &pem) { + using namespace tls; + ctx_ = create_server_context(); + if (ctx_) { + if (!set_server_cert_pem(ctx_, pem.cert_pem, pem.key_pem, + pem.private_key_password)) { + last_ssl_error_ = static_cast(get_error()); + free_context(ctx_); + ctx_ = nullptr; + } else if (pem.client_ca_pem && pem.client_ca_pem_len > 0) { + if (!load_ca_pem(ctx_, pem.client_ca_pem, pem.client_ca_pem_len)) { + last_ssl_error_ = static_cast(get_error()); + free_context(ctx_); + ctx_ = nullptr; + } else { + set_verify_client(ctx_, true); + } + } + } +} + +inline SSLServer::SSLServer(const tls::ContextSetupCallback &setup_callback) { + using namespace tls; + ctx_ = create_server_context(); + if (ctx_) { + if (!setup_callback(ctx_)) { + free_context(ctx_); + ctx_ = nullptr; + } + } +} + +inline SSLServer::~SSLServer() { + if (ctx_) { tls::free_context(ctx_); } +} + +inline bool SSLServer::is_valid() const { return ctx_ != nullptr; } + +inline bool SSLServer::process_and_close_socket(socket_t sock) { + using namespace tls; + + // Create TLS session with mutex protection + session_t session = nullptr; + { + std::lock_guard guard(ctx_mutex_); + session = create_session(static_cast(ctx_), sock); + } + + if (!session) { + last_ssl_error_ = static_cast(get_error()); + detail::shutdown_socket(sock); + detail::close_socket(sock); + return false; + } + + // Use scope_exit to ensure cleanup on all paths (including exceptions) + bool handshake_done = false; + bool ret = false; + bool websocket_upgraded = false; + auto cleanup = detail::scope_exit([&] { + if (handshake_done) { shutdown(session, !websocket_upgraded && ret); } + free_session(session); + detail::shutdown_socket(sock); + detail::close_socket(sock); + }); + + // Perform TLS accept handshake with timeout + TlsError tls_err; + if (!accept_nonblocking(session, sock, read_timeout_sec_, read_timeout_usec_, + &tls_err)) { +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + // Map TlsError to legacy ssl_error for backward compatibility + if (tls_err.code == ErrorCode::WantRead) { + last_ssl_error_ = SSL_ERROR_WANT_READ; + } else if (tls_err.code == ErrorCode::WantWrite) { + last_ssl_error_ = SSL_ERROR_WANT_WRITE; + } else { + last_ssl_error_ = SSL_ERROR_SSL; + } +#else + last_ssl_error_ = static_cast(get_error()); +#endif + return false; + } + + handshake_done = true; + + std::string remote_addr; + int remote_port = 0; + detail::get_remote_ip_and_port(sock, remote_addr, remote_port); + + std::string local_addr; + int local_port = 0; + detail::get_local_ip_and_port(sock, local_addr, local_port); + + ret = detail::process_server_socket_ssl( + svr_sock_, session, sock, keep_alive_max_count_, keep_alive_timeout_sec_, + read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, + write_timeout_usec_, + [&](Stream &strm, bool close_connection, bool &connection_closed) { + return process_request( + strm, remote_addr, remote_port, local_addr, local_port, + close_connection, connection_closed, + [&](Request &req) { req.ssl = session; }, &websocket_upgraded); + }); + + return ret; +} + +inline bool SSLServer::update_certs_pem(const char *cert_pem, + const char *key_pem, + const char *client_ca_pem, + const char *password) { + if (!ctx_) { return false; } + std::lock_guard guard(ctx_mutex_); + if (!tls::update_server_cert(ctx_, cert_pem, key_pem, password)) { + return false; + } + if (client_ca_pem) { + return tls::update_server_client_ca(ctx_, client_ca_pem); + } + return true; +} + +// SSL HTTP client implementation +inline SSLClient::~SSLClient() { + if (ctx_) { tls::free_context(ctx_); } + // Make sure to shut down SSL since shutdown_ssl will resolve to the + // base function rather than the derived function once we get to the + // base class destructor, and won't free the SSL (causing a leak). + shutdown_ssl_impl(socket_, true); +} + +inline bool SSLClient::is_valid() const { return ctx_ != nullptr; } + +inline void SSLClient::shutdown_ssl(Socket &socket, bool shutdown_gracefully) { + shutdown_ssl_impl(socket, shutdown_gracefully); +} + +inline void SSLClient::shutdown_ssl_impl(Socket &socket, + bool shutdown_gracefully) { + if (socket.sock == INVALID_SOCKET) { + assert(socket.ssl == nullptr); + return; + } + if (socket.ssl) { + tls::shutdown(socket.ssl, shutdown_gracefully); + { + std::lock_guard guard(ctx_mutex_); + tls::free_session(socket.ssl); + } + socket.ssl = nullptr; + } + assert(socket.ssl == nullptr); +} + +inline bool SSLClient::process_socket( + const Socket &socket, + std::chrono::time_point start_time, + std::function callback) { + assert(socket.ssl); + return detail::process_client_socket_ssl( + socket.ssl, socket.sock, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_, max_timeout_msec_, start_time, + std::move(callback)); +} + +inline bool SSLClient::is_ssl() const { return true; } + +inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) { + if (!is_valid()) { + error = Error::SSLConnection; + return false; + } + return ClientImpl::create_and_connect_socket(socket, error); +} + +inline bool SSLClient::setup_proxy_connection( + Socket &socket, + std::chrono::time_point start_time, + Response &res, bool &success, Error &error) { + if (!is_proxy_enabled_for_host(host_)) { return true; } + + if (!connect_with_proxy(socket, start_time, res, success, error)) { + return false; + } + + if (!initialize_ssl(socket, error)) { + success = false; + return false; + } + + return true; +} + +// Assumes that socket_mutex_ is locked and that there are no requests in +// flight +inline bool SSLClient::connect_with_proxy( + Socket &socket, + std::chrono::time_point start_time, + Response &res, bool &success, Error &error) { + success = true; + Response proxy_res; + if (!detail::process_client_socket( + socket.sock, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_, max_timeout_msec_, + start_time, [&](Stream &strm) { + Request req2; + req2.method = "CONNECT"; + req2.path = + detail::make_host_and_port_string_always_port(host_, port_); + if (max_timeout_msec_ > 0) { + req2.start_time_ = std::chrono::steady_clock::now(); + } + return process_request(strm, req2, proxy_res, false, error); + })) { + // Thread-safe to close everything because we are assuming there are no + // requests in flight + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + success = false; + return false; + } + + if (proxy_res.status == StatusCode::ProxyAuthenticationRequired_407) { + if (!proxy_digest_auth_username_.empty() && + !proxy_digest_auth_password_.empty()) { + std::map auth; + if (detail::parse_www_authenticate(proxy_res, auth, true)) { + // Close the current socket and create a new one for the authenticated + // request + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + + // Create a new socket for the authenticated CONNECT request + if (!ensure_socket_connection(socket, error)) { + success = false; + output_error_log(error, nullptr); + return false; + } + + proxy_res = Response(); + if (!detail::process_client_socket( + socket.sock, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_, max_timeout_msec_, + start_time, [&](Stream &strm) { + Request req3; + req3.method = "CONNECT"; + req3.path = detail::make_host_and_port_string_always_port( + host_, port_); + req3.headers.insert(detail::make_digest_authentication_header( + req3, auth, 1, detail::random_string(10), + proxy_digest_auth_username_, proxy_digest_auth_password_, + true)); + if (max_timeout_msec_ > 0) { + req3.start_time_ = std::chrono::steady_clock::now(); + } + return process_request(strm, req3, proxy_res, false, error); + })) { + // Thread-safe to close everything because we are assuming there are + // no requests in flight + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + success = false; + return false; + } + } + } + } + + // If status code is not 200, proxy request is failed. + // Set error to ProxyConnection and return proxy response + // as the response of the request + if (proxy_res.status != StatusCode::OK_200) { + error = Error::ProxyConnection; + output_error_log(error, nullptr); + res = std::move(proxy_res); + // Thread-safe to close everything because we are assuming there are + // no requests in flight + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + return false; + } + + return true; +} + +inline bool SSLClient::ensure_socket_connection(Socket &socket, Error &error) { + if (!ClientImpl::ensure_socket_connection(socket, error)) { return false; } + + if (is_proxy_enabled_for_host(host_)) { return true; } + + if (!initialize_ssl(socket, error)) { + shutdown_socket(socket); + close_socket(socket); + return false; + } + + return true; +} + +// SSL HTTP client implementation +inline SSLClient::SSLClient(const std::string &host) + : SSLClient(host, 443, std::string(), std::string()) {} + +inline SSLClient::SSLClient(const std::string &host, int port) + : SSLClient(host, port, std::string(), std::string()) {} + +inline void SSLClient::init_ctx() { + ctx_ = tls::create_client_context(); + if (ctx_) { tls::set_min_version(ctx_, tls::Version::TLS1_2); } +} + +inline void SSLClient::reset_ctx_on_error() { + last_backend_error_ = tls::get_error(); + tls::free_context(ctx_); + ctx_ = nullptr; +} + +inline SSLClient::SSLClient(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path, + const std::string &private_key_password) + : ClientImpl(host, port, client_cert_path, client_key_path) { + init_ctx(); + if (!ctx_) { return; } + + if (!client_cert_path.empty() && !client_key_path.empty()) { + const char *password = + private_key_password.empty() ? nullptr : private_key_password.c_str(); + if (!tls::set_client_cert_file(ctx_, client_cert_path.c_str(), + client_key_path.c_str(), password)) { + reset_ctx_on_error(); + } + } +} + +inline SSLClient::SSLClient(const std::string &host, int port, + const PemMemory &pem) + : ClientImpl(host, port) { + init_ctx(); + if (!ctx_) { return; } + + if (pem.cert_pem && pem.key_pem) { + if (!tls::set_client_cert_pem(ctx_, pem.cert_pem, pem.key_pem, + pem.private_key_password)) { + reset_ctx_on_error(); + } + } +} + +inline void SSLClient::set_ca_cert_store(tls::ca_store_t ca_cert_store) { + if (ca_cert_store && ctx_) { + // set_ca_store takes ownership of ca_cert_store + tls::set_ca_store(ctx_, ca_cert_store); + ca_cert_store_set_ = true; + } else if (ca_cert_store) { + tls::free_ca_store(ca_cert_store); + } +} + +inline void +SSLClient::set_server_certificate_verifier(tls::VerifyCallback verifier) { + if (!ctx_) { return; } + tls::set_verify_callback(ctx_, verifier); +} + +inline void SSLClient::set_session_verifier( + std::function verifier) { + session_verifier_ = std::move(verifier); +} + +#ifdef CPPHTTPLIB_WINDOWS_AUTOMATIC_ROOT_CERTIFICATES_UPDATE +inline void SSLClient::enable_windows_certificate_verification(bool enabled) { + enable_windows_cert_verification_ = enabled; +} +#endif + +inline void SSLClient::load_ca_cert_store(const char *ca_cert, + std::size_t size) { + if (ctx_ && ca_cert && size > 0) { + ca_cert_pem_.assign(ca_cert, size); // Store for redirect transfer + tls::load_ca_pem(ctx_, ca_cert, size); + } +} + +inline bool SSLClient::load_certs() { + auto ret = true; + + std::call_once(initialize_cert_, [&]() { + std::lock_guard guard(ctx_mutex_); + + ret = detail::load_client_ca_config( + ctx_, ca_cert_file_path_, ca_cert_dir_path_, + !ca_cert_pem_.empty() || ca_cert_store_set_, system_ca_mode_, + last_backend_error_); + }); + + return ret; +} + +inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) { + using namespace tls; + + // Load CA certificates if server verification is enabled + if (server_certificate_verification_) { + if (!load_certs()) { + error = Error::SSLLoadingCerts; + output_error_log(error, nullptr); + return false; + } + } + + bool is_ip = detail::is_ip_address(host_); + +#if defined(CPPHTTPLIB_MBEDTLS_SUPPORT) || defined(CPPHTTPLIB_WOLFSSL_SUPPORT) + // MbedTLS/wolfSSL need explicit verification mode (OpenSSL uses + // SSL_VERIFY_NONE by default and performs all verification post-handshake). + // Chain verification happens during the handshake even for IP hosts; the + // certificate identity is verified post-handshake via verify_hostname(). + set_verify_client(ctx_, server_certificate_verification_); +#endif + + // Create TLS session + session_t session = nullptr; + { + std::lock_guard guard(ctx_mutex_); + session = create_session(ctx_, socket.sock); + } + + if (!session) { + error = Error::SSLConnection; + last_backend_error_ = get_error(); + return false; + } + + // Use scope_exit to ensure session is freed on error paths + bool success = false; + auto session_guard = detail::scope_exit([&] { + if (!success) { free_session(session); } + }); + + // Set SNI extension (skip for IP addresses per RFC 6066). + // On MbedTLS, set_sni also enables hostname verification internally. + // On OpenSSL, set_sni only sets SNI; verification is done post-handshake. + if (!is_ip) { + if (!set_sni(session, host_.c_str())) { + error = Error::SSLConnection; + last_backend_error_ = get_error(); + return false; + } + } + + // Perform non-blocking TLS handshake with timeout + TlsError tls_err; + if (!connect_nonblocking(session, socket.sock, connection_timeout_sec_, + connection_timeout_usec_, &tls_err)) { + last_ssl_error_ = static_cast(tls_err.code); + last_backend_error_ = tls_err.backend_code; + if (tls_err.code == ErrorCode::CertVerifyFailed) { + error = Error::SSLServerVerification; + } else if (tls_err.code == ErrorCode::HostnameMismatch) { + error = Error::SSLServerHostnameVerification; + } else { + error = Error::SSLConnection; + } + output_error_log(error, nullptr); + return false; + } + + // Post-handshake session verifier callback + auto verification_status = SSLVerifierResponse::NoDecisionMade; + if (session_verifier_) { verification_status = session_verifier_(session); } + + if (verification_status == SSLVerifierResponse::CertificateRejected) { + last_backend_error_ = get_error(); + error = Error::SSLServerVerification; + output_error_log(error, nullptr); + return false; + } + + // Default server certificate verification + if (verification_status == SSLVerifierResponse::NoDecisionMade && + server_certificate_verification_) { + verify_result_ = tls::get_verify_result(session); + if (verify_result_ != 0) { + last_backend_error_ = static_cast(verify_result_); + error = Error::SSLServerVerification; + output_error_log(error, nullptr); + return false; + } + + auto server_cert = get_peer_cert(session); + if (!server_cert) { + last_backend_error_ = get_error(); + error = Error::SSLServerVerification; + output_error_log(error, nullptr); + return false; + } + auto cert_guard = detail::scope_exit([&] { free_cert(server_cert); }); + + // Hostname verification (post-handshake for all cases). + // On OpenSSL, verification is always post-handshake (SSL_VERIFY_NONE). + // On MbedTLS, set_sni already enabled hostname verification during + // handshake for non-IP hosts, but this check is still needed for IP + // addresses where SNI is not set. + if (server_hostname_verification_) { + if (!verify_hostname(server_cert, host_.c_str())) { + last_backend_error_ = hostname_mismatch_code(); + error = Error::SSLServerHostnameVerification; + output_error_log(error, nullptr); + return false; + } + } + +#ifdef CPPHTTPLIB_WINDOWS_AUTOMATIC_ROOT_CERTIFICATES_UPDATE + // Additional Windows Schannel verification. + // This provides real-time certificate validation with Windows Update + // integration, working with both OpenSSL and MbedTLS backends. + // Skip when a custom CA cert is specified, as the Windows certificate + // store would not know about user-provided CA certificates. Also skip + // when system CA trust is explicitly disabled. + if (enable_windows_cert_verification_ && + system_ca_mode_ != SystemCAMode::Disabled && + ca_cert_file_path_.empty() && ca_cert_dir_path_.empty() && + ca_cert_pem_.empty() && !ca_cert_store_set_) { + std::vector der; + if (get_cert_der(server_cert, der)) { + uint64_t wincrypt_error = 0; + if (!detail::verify_cert_with_windows_schannel( + der, host_, server_hostname_verification_, wincrypt_error)) { + last_backend_error_ = wincrypt_error; + error = Error::SSLServerVerification; + output_error_log(error, nullptr); + return false; + } + } + } +#endif + } + + success = true; + socket.ssl = session; + return true; +} + +inline void Client::set_digest_auth(const std::string &username, + const std::string &password) { + cli_->set_digest_auth(username, password); +} + +inline void Client::set_proxy_digest_auth(const std::string &username, + const std::string &password) { + cli_->set_proxy_digest_auth(username, password); +} + +inline void Client::enable_server_certificate_verification(bool enabled) { + cli_->enable_server_certificate_verification(enabled); +} + +inline void Client::enable_server_hostname_verification(bool enabled) { + cli_->enable_server_hostname_verification(enabled); +} + +inline void Client::enable_system_ca(bool enabled) { + cli_->enable_system_ca(enabled); +} + +#ifdef CPPHTTPLIB_WINDOWS_AUTOMATIC_ROOT_CERTIFICATES_UPDATE +inline void Client::enable_windows_certificate_verification(bool enabled) { + if (is_ssl_) { + static_cast(*cli_).enable_windows_certificate_verification( + enabled); + } +} +#endif + +inline void Client::set_ca_cert_path(const std::string &ca_cert_file_path, + const std::string &ca_cert_dir_path) { + cli_->set_ca_cert_path(ca_cert_file_path, ca_cert_dir_path); +} + +inline void Client::set_ca_cert_store(tls::ca_store_t ca_cert_store) { + if (is_ssl_) { + static_cast(*cli_).set_ca_cert_store(ca_cert_store); + } else if (ca_cert_store) { + tls::free_ca_store(ca_cert_store); + } +} + +inline void Client::load_ca_cert_store(const char *ca_cert, std::size_t size) { + if (is_ssl_) { + // Use the PEM-based path so the CA data is retained for redirect transfer + static_cast(*cli_).load_ca_cert_store(ca_cert, size); + } +} + +inline void +Client::set_server_certificate_verifier(tls::VerifyCallback verifier) { + if (is_ssl_) { + static_cast(*cli_).set_server_certificate_verifier( + std::move(verifier)); + } +} + +inline void Client::set_session_verifier( + std::function verifier) { + if (is_ssl_) { + static_cast(*cli_).set_session_verifier(std::move(verifier)); + } +} + +inline tls::ctx_t Client::tls_context() const { + if (is_ssl_) { return static_cast(*cli_).tls_context(); } + return nullptr; +} + +#endif // CPPHTTPLIB_SSL_ENABLED + +/* + * Group 7: TLS abstraction layer - Common API + */ + +#ifdef CPPHTTPLIB_SSL_ENABLED + +namespace tls { + +// Helper for PeerCert construction +inline PeerCert get_peer_cert_from_session(const_session_t session) { + return PeerCert(get_peer_cert(session)); +} + +namespace impl { + +inline VerifyCallback &get_verify_callback() { + static thread_local VerifyCallback callback; + return callback; +} + +inline VerifyCallback &get_mbedtls_verify_callback() { + static thread_local VerifyCallback callback; + return callback; +} + +// Check if a string is an IPv4 address +inline bool is_ipv4_address(const std::string &str) { + int dots = 0; + for (char c : str) { + if (c == '.') { + dots++; + } else if (!isdigit(static_cast(c))) { + return false; + } + } + return dots == 3; +} + +// Parse IPv4 address string to bytes +inline bool parse_ipv4(const std::string &str, unsigned char *out) { + const char *p = str.c_str(); + for (int i = 0; i < 4; i++) { + if (i > 0) { + if (*p != '.') { return false; } + p++; + } + int val = 0; + int digits = 0; + while (*p >= '0' && *p <= '9') { + val = val * 10 + (*p - '0'); + if (val > 255) { return false; } + p++; + digits++; + } + if (digits == 0) { return false; } + // Reject leading zeros (e.g., "01.002.03.04") to prevent ambiguity + if (digits > 1 && *(p - digits) == '0') { return false; } + out[i] = static_cast(val); + } + return *p == '\0'; +} + +// Parse an IP literal (IPv4 or IPv6) into raw network-order bytes. +// `out` must have room for at least 16 bytes. Returns the address length +// (4 for IPv4, 16 for IPv6) on success, or 0 if the string is not an IP +// literal. Used to match a host against iPAddress SANs the same way the +// OpenSSL backend does via X509_check_ip. +inline size_t parse_ip_address(const std::string &str, unsigned char *out) { + if (is_ipv4_address(str)) { return parse_ipv4(str, out) ? 4 : 0; } + struct in6_addr addr6 = {}; + if (inet_pton(AF_INET6, str.c_str(), &addr6) == 1) { + memcpy(out, &addr6, 16); + return 16; + } + return 0; +} + +#ifdef _WIN32 +// Enumerate Windows system certificates and call callback with DER data +template +inline bool enumerate_windows_system_certs(Callback cb) { + bool loaded = false; + static const wchar_t *store_names[] = {L"ROOT", L"CA"}; + for (auto store_name : store_names) { + HCERTSTORE hStore = CertOpenSystemStoreW(0, store_name); + if (hStore) { + PCCERT_CONTEXT pContext = nullptr; + while ((pContext = CertEnumCertificatesInStore(hStore, pContext)) != + nullptr) { + if (cb(pContext->pbCertEncoded, pContext->cbCertEncoded)) { + loaded = true; + } + } + CertCloseStore(hStore, 0); + } + } + return loaded; +} +#endif + +#ifdef CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN +// Enumerate macOS Keychain certificates and call callback with DER data +template +inline bool enumerate_macos_keychain_certs(Callback cb) { + bool loaded = false; + const SecTrustSettingsDomain domains[] = { + kSecTrustSettingsDomainSystem, + kSecTrustSettingsDomainAdmin, + kSecTrustSettingsDomainUser, + }; + for (auto domain : domains) { + CFArrayRef certs = nullptr; + OSStatus status = SecTrustSettingsCopyCertificates(domain, &certs); + if (status != errSecSuccess || !certs) { + if (certs) CFRelease(certs); + continue; + } + CFIndex count = CFArrayGetCount(certs); + for (CFIndex i = 0; i < count; i++) { + SecCertificateRef cert = + (SecCertificateRef)CFArrayGetValueAtIndex(certs, i); + CFDataRef data = SecCertificateCopyData(cert); + if (data) { + if (cb(CFDataGetBytePtr(data), + static_cast(CFDataGetLength(data)))) { + loaded = true; + } + CFRelease(data); + } + } + CFRelease(certs); + } + return loaded; +} +#endif + +#if !defined(_WIN32) && !(defined(__APPLE__) && \ + defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN)) +// Common CA certificate file paths on Linux/Unix +inline const char **system_ca_paths() { + static const char *paths[] = { + "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu + "/etc/pki/tls/certs/ca-bundle.crt", // RHEL/CentOS + "/etc/ssl/ca-bundle.pem", // OpenSUSE + "/etc/pki/tls/cacert.pem", // OpenELEC + "/etc/ssl/cert.pem", // Alpine, FreeBSD + nullptr}; + return paths; +} + +// Common CA certificate directory paths on Linux/Unix +inline const char **system_ca_dirs() { + static const char *dirs[] = {"/etc/ssl/certs", // Debian/Ubuntu + "/etc/pki/tls/certs", // RHEL/CentOS + "/usr/share/ca-certificates", // Other + nullptr}; + return dirs; +} +#endif + +} // namespace impl + +inline bool set_client_ca_file(ctx_t ctx, const char *ca_file, + const char *ca_dir) { + if (!ctx) { return false; } + + bool success = true; + if (ca_file && *ca_file) { + if (!load_ca_file(ctx, ca_file)) { success = false; } + } + if (ca_dir && *ca_dir) { + if (!load_ca_dir(ctx, ca_dir)) { success = false; } + } + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + // Set CA list for client certificate request (CertificateRequest message) + if (ca_file && *ca_file) { + auto list = SSL_load_client_CA_file(ca_file); + if (list) { SSL_CTX_set_client_CA_list(static_cast(ctx), list); } + } +#endif + + return success; +} + +inline bool set_server_cert_pem(ctx_t ctx, const char *cert, const char *key, + const char *password) { + return set_client_cert_pem(ctx, cert, key, password); +} + +inline bool set_server_cert_file(ctx_t ctx, const char *cert_path, + const char *key_path, const char *password) { + return set_client_cert_file(ctx, cert_path, key_path, password); +} + +// PeerCert implementation +inline PeerCert::PeerCert() = default; + +inline PeerCert::PeerCert(cert_t cert) : cert_(cert) {} + +inline PeerCert::PeerCert(PeerCert &&other) noexcept : cert_(other.cert_) { + other.cert_ = nullptr; +} + +inline PeerCert &PeerCert::operator=(PeerCert &&other) noexcept { + if (this != &other) { + if (cert_) { free_cert(cert_); } + cert_ = other.cert_; + other.cert_ = nullptr; + } + return *this; +} + +inline PeerCert::~PeerCert() { + if (cert_) { free_cert(cert_); } +} + +inline PeerCert::operator bool() const { return cert_ != nullptr; } + +inline std::string PeerCert::subject_cn() const { + return cert_ ? get_cert_subject_cn(cert_) : std::string(); +} + +inline std::string PeerCert::issuer_name() const { + return cert_ ? get_cert_issuer_name(cert_) : std::string(); +} + +inline bool PeerCert::check_hostname(const char *hostname) const { + return cert_ ? verify_hostname(cert_, hostname) : false; +} + +inline std::vector PeerCert::sans() const { + std::vector result; + if (cert_) { get_cert_sans(cert_, result); } + return result; +} + +inline bool PeerCert::validity(time_t ¬_before, time_t ¬_after) const { + return cert_ ? get_cert_validity(cert_, not_before, not_after) : false; +} + +inline std::string PeerCert::serial() const { + return cert_ ? get_cert_serial(cert_) : std::string(); +} + +// VerifyContext method implementations +inline std::string VerifyContext::subject_cn() const { + return cert ? get_cert_subject_cn(cert) : std::string(); +} + +inline std::string VerifyContext::issuer_name() const { + return cert ? get_cert_issuer_name(cert) : std::string(); +} + +inline bool VerifyContext::check_hostname(const char *hostname) const { + return cert ? verify_hostname(cert, hostname) : false; +} + +inline std::vector VerifyContext::sans() const { + std::vector result; + if (cert) { get_cert_sans(cert, result); } + return result; +} + +inline bool VerifyContext::validity(time_t ¬_before, + time_t ¬_after) const { + return cert ? get_cert_validity(cert, not_before, not_after) : false; +} + +inline std::string VerifyContext::serial() const { + return cert ? get_cert_serial(cert) : std::string(); +} + +// TlsError static method implementation +inline std::string TlsError::verify_error_to_string(long error_code) { + return verify_error_string(error_code); +} + +} // namespace tls + +// Request::peer_cert() implementation +inline tls::PeerCert Request::peer_cert() const { + return tls::get_peer_cert_from_session(ssl); +} + +// Request::sni() implementation +inline std::string Request::sni() const { + if (!ssl) { return std::string(); } + const char *s = tls::get_sni(ssl); + return s ? std::string(s) : std::string(); +} + +#endif // CPPHTTPLIB_SSL_ENABLED + +/* + * Group 8: TLS abstraction layer - OpenSSL backend + */ + +/* + * OpenSSL Backend Implementation + */ + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +namespace tls { + +namespace impl { + +// Helper to map OpenSSL SSL_get_error to ErrorCode +inline ErrorCode map_ssl_error(int ssl_error, int &out_errno) { + switch (ssl_error) { + case SSL_ERROR_NONE: return ErrorCode::Success; + case SSL_ERROR_WANT_READ: return ErrorCode::WantRead; + case SSL_ERROR_WANT_WRITE: return ErrorCode::WantWrite; + case SSL_ERROR_ZERO_RETURN: return ErrorCode::PeerClosed; + case SSL_ERROR_SYSCALL: out_errno = errno; return ErrorCode::SyscallError; + case SSL_ERROR_SSL: + default: return ErrorCode::Fatal; + } +} + +// Helper: Create client CA list from PEM string +// Returns a new STACK_OF(X509_NAME)* or nullptr on failure +// Caller takes ownership of returned list +inline STACK_OF(X509_NAME) * + create_client_ca_list_from_pem(const char *ca_pem) { + if (!ca_pem) { return nullptr; } + + auto ca_list = sk_X509_NAME_new_null(); + if (!ca_list) { return nullptr; } + + BIO *bio = BIO_new_mem_buf(ca_pem, -1); + if (!bio) { + sk_X509_NAME_pop_free(ca_list, X509_NAME_free); + return nullptr; + } + + X509 *cert = nullptr; + while ((cert = PEM_read_bio_X509(bio, nullptr, nullptr, nullptr)) != + nullptr) { + const X509_NAME *name = X509_get_subject_name(cert); + if (name) { + sk_X509_NAME_push(ca_list, X509_NAME_dup(const_cast(name))); + } + X509_free(cert); + } + BIO_free(bio); + + return ca_list; +} + +// OpenSSL verify callback wrapper +inline int openssl_verify_callback(int preverify_ok, X509_STORE_CTX *ctx) { + auto &callback = get_verify_callback(); + if (!callback) { return preverify_ok; } + + // Get SSL object from X509_STORE_CTX + auto ssl = static_cast( + X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx())); + if (!ssl) { return preverify_ok; } + + // Get current certificate and depth + auto cert = X509_STORE_CTX_get_current_cert(ctx); + int depth = X509_STORE_CTX_get_error_depth(ctx); + int error = X509_STORE_CTX_get_error(ctx); + + // Build context + VerifyContext verify_ctx; + verify_ctx.session = static_cast(ssl); + verify_ctx.cert = static_cast(cert); + verify_ctx.depth = depth; + verify_ctx.preverify_ok = (preverify_ok != 0); + verify_ctx.error_code = error; + verify_ctx.error_string = + (error != X509_V_OK) ? X509_verify_cert_error_string(error) : nullptr; + + return callback(verify_ctx) ? 1 : 0; +} + +// X509_STORE_get0_objects is deprecated since OpenSSL 4.0 because it is not +// thread-safe; X509_STORE_get1_objects (OpenSSL 3.3+) returns a snapshot +// that must be released with release_store_objects +#if !defined(OPENSSL_IS_BORINGSSL) && !defined(LIBRESSL_VERSION_NUMBER) && \ + OPENSSL_VERSION_NUMBER >= 0x30300000L +#define CPPHTTPLIB_HAS_X509_STORE_GET1_OBJECTS +#endif + +inline STACK_OF(X509_OBJECT) * get_store_objects(X509_STORE *store) { +#ifdef CPPHTTPLIB_HAS_X509_STORE_GET1_OBJECTS + return X509_STORE_get1_objects(store); +#else + return X509_STORE_get0_objects(store); +#endif +} + +inline void release_store_objects(STACK_OF(X509_OBJECT) * objs) { +#ifdef CPPHTTPLIB_HAS_X509_STORE_GET1_OBJECTS + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); +#else + (void)objs; // get0 variant returns an internal pointer; nothing to free +#endif +} + +} // namespace impl + +inline ctx_t create_client_context() { + SSL_CTX *ctx = SSL_CTX_new(TLS_client_method()); + if (ctx) { + // Disable auto-retry to properly handle non-blocking I/O + SSL_CTX_clear_mode(ctx, SSL_MODE_AUTO_RETRY); + // Set minimum TLS version + SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION); + } + return static_cast(ctx); +} + +inline void free_context(ctx_t ctx) { + if (ctx) { SSL_CTX_free(static_cast(ctx)); } +} + +inline bool set_min_version(ctx_t ctx, Version version) { + if (!ctx) return false; + return SSL_CTX_set_min_proto_version(static_cast(ctx), + static_cast(version)) == 1; +} + +inline bool load_ca_pem(ctx_t ctx, const char *pem, size_t len) { + if (!ctx || !pem || len == 0) return false; + + auto ssl_ctx = static_cast(ctx); + auto store = SSL_CTX_get_cert_store(ssl_ctx); + if (!store) return false; + + auto bio = BIO_new_mem_buf(pem, static_cast(len)); + if (!bio) return false; + + bool ok = true; + X509 *cert = nullptr; + while ((cert = PEM_read_bio_X509(bio, nullptr, nullptr, nullptr)) != + nullptr) { + if (X509_STORE_add_cert(store, cert) != 1) { + // Ignore duplicate errors + auto err = ERR_peek_last_error(); + if (ERR_GET_REASON(err) != X509_R_CERT_ALREADY_IN_HASH_TABLE) { + ok = false; + } + } + X509_free(cert); + if (!ok) break; + } + BIO_free(bio); + + // Clear any "no more certificates" errors + ERR_clear_error(); + return ok; +} + +inline bool load_ca_file(ctx_t ctx, const char *file_path) { + if (!ctx || !file_path) return false; + return SSL_CTX_load_verify_locations(static_cast(ctx), file_path, + nullptr) == 1; +} + +inline bool load_ca_dir(ctx_t ctx, const char *dir_path) { + if (!ctx || !dir_path) return false; + return SSL_CTX_load_verify_locations(static_cast(ctx), nullptr, + dir_path) == 1; +} + +inline bool load_system_certs(ctx_t ctx) { + if (!ctx) return false; + auto ssl_ctx = static_cast(ctx); + +#ifdef _WIN32 + // Windows: Load from system certificate store (ROOT and CA) + auto store = SSL_CTX_get_cert_store(ssl_ctx); + if (!store) return false; + + bool loaded_any = false; + static const wchar_t *store_names[] = {L"ROOT", L"CA"}; + for (auto store_name : store_names) { + auto hStore = CertOpenSystemStoreW(NULL, store_name); + if (!hStore) continue; + + PCCERT_CONTEXT pContext = nullptr; + while ((pContext = CertEnumCertificatesInStore(hStore, pContext)) != + nullptr) { + const unsigned char *data = pContext->pbCertEncoded; + auto x509 = d2i_X509(nullptr, &data, pContext->cbCertEncoded); + if (x509) { + if (X509_STORE_add_cert(store, x509) == 1) { loaded_any = true; } + X509_free(x509); + } + } + CertCloseStore(hStore, 0); + } + return loaded_any; + +#elif defined(__APPLE__) +#ifdef CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN + // macOS: Load from Keychain + auto store = SSL_CTX_get_cert_store(ssl_ctx); + if (!store) return false; + + bool loaded_any = false; + const SecTrustSettingsDomain domains[] = { + kSecTrustSettingsDomainSystem, + kSecTrustSettingsDomainAdmin, + kSecTrustSettingsDomainUser, + }; + for (auto domain : domains) { + CFArrayRef certs = nullptr; + if (SecTrustSettingsCopyCertificates(domain, &certs) != errSecSuccess || + !certs) { + if (certs) CFRelease(certs); + continue; + } + auto count = CFArrayGetCount(certs); + for (CFIndex i = 0; i < count; i++) { + auto cert = reinterpret_cast( + const_cast(CFArrayGetValueAtIndex(certs, i))); + CFDataRef der = SecCertificateCopyData(cert); + if (der) { + const unsigned char *data = CFDataGetBytePtr(der); + auto x509 = d2i_X509(nullptr, &data, CFDataGetLength(der)); + if (x509) { + if (X509_STORE_add_cert(store, x509) == 1) { loaded_any = true; } + X509_free(x509); + } + CFRelease(der); + } + } + CFRelease(certs); + } + return loaded_any || SSL_CTX_set_default_verify_paths(ssl_ctx) == 1; +#else + return SSL_CTX_set_default_verify_paths(ssl_ctx) == 1; +#endif + +#else + // Other Unix: use default verify paths + return SSL_CTX_set_default_verify_paths(ssl_ctx) == 1; +#endif +} + +inline bool set_client_cert_pem(ctx_t ctx, const char *cert, const char *key, + const char *password) { + if (!ctx || !cert || !key) return false; + + auto ssl_ctx = static_cast(ctx); + + // Load certificate + auto cert_bio = BIO_new_mem_buf(cert, -1); + if (!cert_bio) return false; + + auto x509 = PEM_read_bio_X509(cert_bio, nullptr, nullptr, nullptr); + BIO_free(cert_bio); + if (!x509) return false; + + auto cert_ok = SSL_CTX_use_certificate(ssl_ctx, x509) == 1; + X509_free(x509); + if (!cert_ok) return false; + + // Load private key + auto key_bio = BIO_new_mem_buf(key, -1); + if (!key_bio) return false; + + auto pkey = PEM_read_bio_PrivateKey(key_bio, nullptr, nullptr, + password ? const_cast(password) + : nullptr); + BIO_free(key_bio); + if (!pkey) return false; + + auto key_ok = SSL_CTX_use_PrivateKey(ssl_ctx, pkey) == 1; + EVP_PKEY_free(pkey); + + return key_ok && SSL_CTX_check_private_key(ssl_ctx) == 1; +} + +inline bool set_client_cert_file(ctx_t ctx, const char *cert_path, + const char *key_path, const char *password) { + if (!ctx || !cert_path || !key_path) return false; + + auto ssl_ctx = static_cast(ctx); + + if (password && password[0] != '\0') { + SSL_CTX_set_default_passwd_cb_userdata( + ssl_ctx, reinterpret_cast(const_cast(password))); + } + + return SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_path) == 1 && + SSL_CTX_use_PrivateKey_file(ssl_ctx, key_path, SSL_FILETYPE_PEM) == 1; +} + +inline ctx_t create_server_context() { + SSL_CTX *ctx = SSL_CTX_new(TLS_server_method()); + if (ctx) { + SSL_CTX_set_options(ctx, SSL_OP_NO_COMPRESSION | + SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION); + SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION); + } + return static_cast(ctx); +} + +inline void set_verify_client(ctx_t ctx, bool require) { + if (!ctx) return; + SSL_CTX_set_verify(static_cast(ctx), + require + ? (SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT) + : SSL_VERIFY_NONE, + nullptr); +} + +inline session_t create_session(ctx_t ctx, socket_t sock) { + if (!ctx || sock == INVALID_SOCKET) return nullptr; + + auto ssl_ctx = static_cast(ctx); + SSL *ssl = SSL_new(ssl_ctx); + if (!ssl) return nullptr; + + // Disable auto-retry for proper non-blocking I/O handling + SSL_clear_mode(ssl, SSL_MODE_AUTO_RETRY); + + auto bio = BIO_new_socket(static_cast(sock), BIO_NOCLOSE); + if (!bio) { + SSL_free(ssl); + return nullptr; + } + + SSL_set_bio(ssl, bio, bio); + return static_cast(ssl); +} + +inline void free_session(session_t session) { + if (session) { SSL_free(static_cast(session)); } +} + +inline bool set_sni(session_t session, const char *hostname) { + if (!session || !hostname) return false; + + auto ssl = static_cast(session); + + // Set SNI (Server Name Indication) only - does not enable verification +#if defined(OPENSSL_IS_BORINGSSL) + return SSL_set_tlsext_host_name(ssl, hostname) == 1; +#else + // Direct call instead of macro to suppress -Wold-style-cast warning + return SSL_ctrl(ssl, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, + static_cast(const_cast(hostname))) == 1; +#endif +} + +inline bool set_hostname(session_t session, const char *hostname) { + if (!session || !hostname) return false; + + auto ssl = static_cast(session); + + // Enable hostname verification + auto param = SSL_get0_param(ssl); + if (!param) return false; + + if (detail::is_ip_address(hostname)) { + // RFC 6066: SNI must not be set for IP addresses; verify against the + // certificate's IP SANs instead of its DNS names + if (X509_VERIFY_PARAM_set1_ip_asc(param, hostname) != 1) { return false; } + } else { + // Set SNI (Server Name Indication) + if (!set_sni(session, hostname)) { return false; } + + X509_VERIFY_PARAM_set_hostflags(param, + X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); + if (X509_VERIFY_PARAM_set1_host(param, hostname, 0) != 1) { return false; } + } + + SSL_set_verify(ssl, SSL_VERIFY_PEER, nullptr); + return true; +} + +inline TlsError connect(session_t session) { + if (!session) { return TlsError(); } + + auto ssl = static_cast(session); + auto ret = SSL_connect(ssl); + + TlsError err; + if (ret == 1) { + err.code = ErrorCode::Success; + } else { + auto ssl_err = SSL_get_error(ssl, ret); + err.code = impl::map_ssl_error(ssl_err, err.sys_errno); + err.backend_code = ERR_get_error(); + } + return err; +} + +inline TlsError accept(session_t session) { + if (!session) { return TlsError(); } + + auto ssl = static_cast(session); + auto ret = SSL_accept(ssl); + + TlsError err; + if (ret == 1) { + err.code = ErrorCode::Success; + } else { + auto ssl_err = SSL_get_error(ssl, ret); + err.code = impl::map_ssl_error(ssl_err, err.sys_errno); + err.backend_code = ERR_get_error(); + } + return err; +} + +inline bool connect_nonblocking(session_t session, socket_t sock, + time_t timeout_sec, time_t timeout_usec, + TlsError *err) { + if (!session) { + if (err) { err->code = ErrorCode::Fatal; } + return false; + } + + auto ssl = static_cast(session); + auto bio = SSL_get_rbio(ssl); + + // Set non-blocking mode for handshake + detail::set_nonblocking(sock, true); + if (bio) { BIO_set_nbio(bio, 1); } + + auto cleanup = detail::scope_exit([&]() { + // Restore blocking mode after handshake + if (bio) { BIO_set_nbio(bio, 0); } + detail::set_nonblocking(sock, false); + }); + + auto res = 0; + while ((res = SSL_connect(ssl)) != 1) { + auto ssl_err = SSL_get_error(ssl, res); + switch (ssl_err) { + case SSL_ERROR_WANT_READ: + if (detail::select_read(sock, timeout_sec, timeout_usec) > 0) { + continue; + } + break; + case SSL_ERROR_WANT_WRITE: + if (detail::select_write(sock, timeout_sec, timeout_usec) > 0) { + continue; + } + break; + default: break; + } + if (err) { + err->code = impl::map_ssl_error(ssl_err, err->sys_errno); + err->backend_code = ERR_get_error(); + } + return false; + } + if (err) { err->code = ErrorCode::Success; } + return true; +} + +inline bool accept_nonblocking(session_t session, socket_t sock, + time_t timeout_sec, time_t timeout_usec, + TlsError *err) { + if (!session) { + if (err) { err->code = ErrorCode::Fatal; } + return false; + } + + auto ssl = static_cast(session); + auto bio = SSL_get_rbio(ssl); + + // Set non-blocking mode for handshake + detail::set_nonblocking(sock, true); + if (bio) { BIO_set_nbio(bio, 1); } + + auto cleanup = detail::scope_exit([&]() { + // Restore blocking mode after handshake + if (bio) { BIO_set_nbio(bio, 0); } + detail::set_nonblocking(sock, false); + }); + + auto res = 0; + while ((res = SSL_accept(ssl)) != 1) { + auto ssl_err = SSL_get_error(ssl, res); + switch (ssl_err) { + case SSL_ERROR_WANT_READ: + if (detail::select_read(sock, timeout_sec, timeout_usec) > 0) { + continue; + } + break; + case SSL_ERROR_WANT_WRITE: + if (detail::select_write(sock, timeout_sec, timeout_usec) > 0) { + continue; + } + break; + default: break; + } + if (err) { + err->code = impl::map_ssl_error(ssl_err, err->sys_errno); + err->backend_code = ERR_get_error(); + } + return false; + } + if (err) { err->code = ErrorCode::Success; } + return true; +} + +inline ssize_t read(session_t session, void *buf, size_t len, TlsError &err) { + if (!session || !buf) { + err.code = ErrorCode::Fatal; + return -1; + } + + auto ssl = static_cast(session); + constexpr auto max_len = + static_cast((std::numeric_limits::max)()); + if (len > max_len) { len = max_len; } + auto ret = SSL_read(ssl, buf, static_cast(len)); + + if (ret > 0) { + err.code = ErrorCode::Success; + return ret; + } + + auto ssl_err = SSL_get_error(ssl, ret); + err.code = impl::map_ssl_error(ssl_err, err.sys_errno); + if (err.code == ErrorCode::PeerClosed) { + return 0; + } // Gracefully handle the peer closed state. + if (err.code == ErrorCode::Fatal) { err.backend_code = ERR_get_error(); } + return -1; +} + +inline ssize_t write(session_t session, const void *buf, size_t len, + TlsError &err) { + if (!session || !buf) { + err.code = ErrorCode::Fatal; + return -1; + } + + auto ssl = static_cast(session); + auto ret = SSL_write(ssl, buf, static_cast(len)); + + if (ret > 0) { + err.code = ErrorCode::Success; + return ret; + } + + auto ssl_err = SSL_get_error(ssl, ret); + err.code = impl::map_ssl_error(ssl_err, err.sys_errno); + if (err.code == ErrorCode::Fatal) { err.backend_code = ERR_get_error(); } + return -1; +} + +inline int pending(const_session_t session) { + if (!session) return 0; + return SSL_pending(static_cast(const_cast(session))); +} + +inline void shutdown(session_t session, bool graceful) { + if (!session) return; + + auto ssl = static_cast(session); + if (graceful) { + // First call sends close_notify + if (SSL_shutdown(ssl) == 0) { + // Second call waits for peer's close_notify + SSL_shutdown(ssl); + } + } +} + +inline bool is_peer_closed(session_t session, socket_t sock) { + if (!session) return true; + + // Temporarily set socket to non-blocking to avoid blocking on SSL_peek + detail::set_nonblocking(sock, true); + auto se = detail::scope_exit([&]() { detail::set_nonblocking(sock, false); }); + + auto ssl = static_cast(session); + char buf; + auto ret = SSL_peek(ssl, &buf, 1); + if (ret > 0) return false; + + auto err = SSL_get_error(ssl, ret); + return err == SSL_ERROR_ZERO_RETURN; +} + +inline cert_t get_peer_cert(const_session_t session) { + if (!session) return nullptr; + return static_cast(SSL_get1_peer_certificate( + static_cast(const_cast(session)))); +} + +inline void free_cert(cert_t cert) { + if (cert) { X509_free(static_cast(cert)); } +} + +inline bool verify_hostname(cert_t cert, const char *hostname) { + if (!cert || !hostname) return false; + + auto x509 = static_cast(cert); + + // Use X509_check_ip_asc for IP addresses, X509_check_host for DNS names + if (detail::is_ip_address(hostname)) { + return X509_check_ip_asc(x509, hostname, 0) == 1; + } + return X509_check_host(x509, hostname, strlen(hostname), 0, nullptr) == 1; +} + +inline uint64_t hostname_mismatch_code() { + return static_cast(X509_V_ERR_HOSTNAME_MISMATCH); +} + +inline long get_verify_result(const_session_t session) { + if (!session) return X509_V_ERR_UNSPECIFIED; + return SSL_get_verify_result(static_cast(const_cast(session))); +} + +inline std::string get_cert_subject_cn(cert_t cert) { + if (!cert) return ""; + auto x509 = static_cast(cert); + auto subject_name = X509_get_subject_name(x509); + if (!subject_name) return ""; + + // X509_NAME_get_text_by_NID is deprecated since OpenSSL 4.0 + auto idx = X509_NAME_get_index_by_NID(subject_name, NID_commonName, -1); + if (idx < 0) return ""; + + auto entry = X509_NAME_get_entry(subject_name, idx); + if (!entry) return ""; + + auto data = X509_NAME_ENTRY_get_data(entry); + if (!data) return ""; + + return std::string( + reinterpret_cast(ASN1_STRING_get0_data(data)), + static_cast(ASN1_STRING_length(data))); +} + +inline std::string get_cert_issuer_name(cert_t cert) { + if (!cert) return ""; + auto x509 = static_cast(cert); + auto issuer_name = X509_get_issuer_name(x509); + if (!issuer_name) return ""; + + char buf[256]; + X509_NAME_oneline(issuer_name, buf, sizeof(buf)); + return std::string(buf); +} + +inline bool get_cert_sans(cert_t cert, std::vector &sans) { + sans.clear(); + if (!cert) return false; + auto x509 = static_cast(cert); + + auto names = static_cast( + X509_get_ext_d2i(x509, NID_subject_alt_name, nullptr, nullptr)); + if (!names) return true; // No SANs is valid + + auto count = sk_GENERAL_NAME_num(names); + for (decltype(count) i = 0; i < count; i++) { + auto gen = sk_GENERAL_NAME_value(names, i); + if (!gen) continue; + + SanEntry entry; + switch (gen->type) { + case GEN_DNS: + entry.type = SanType::DNS; + if (gen->d.dNSName) { + entry.value = std::string( + reinterpret_cast( + ASN1_STRING_get0_data(gen->d.dNSName)), + static_cast(ASN1_STRING_length(gen->d.dNSName))); + } + break; + case GEN_IPADD: + entry.type = SanType::IP; + if (gen->d.iPAddress) { + auto data = ASN1_STRING_get0_data(gen->d.iPAddress); + auto len = ASN1_STRING_length(gen->d.iPAddress); + if (len == 4) { + // IPv4 + char buf[INET_ADDRSTRLEN]; + inet_ntop(AF_INET, data, buf, sizeof(buf)); + entry.value = buf; + } else if (len == 16) { + // IPv6 + char buf[INET6_ADDRSTRLEN]; + inet_ntop(AF_INET6, data, buf, sizeof(buf)); + entry.value = buf; + } + } + break; + case GEN_EMAIL: + entry.type = SanType::EMAIL; + if (gen->d.rfc822Name) { + entry.value = std::string( + reinterpret_cast( + ASN1_STRING_get0_data(gen->d.rfc822Name)), + static_cast(ASN1_STRING_length(gen->d.rfc822Name))); + } + break; + case GEN_URI: + entry.type = SanType::URI; + if (gen->d.uniformResourceIdentifier) { + entry.value = std::string( + reinterpret_cast( + ASN1_STRING_get0_data(gen->d.uniformResourceIdentifier)), + static_cast( + ASN1_STRING_length(gen->d.uniformResourceIdentifier))); + } + break; + default: entry.type = SanType::OTHER; break; + } + + if (!entry.value.empty()) { sans.push_back(std::move(entry)); } + } + + GENERAL_NAMES_free(names); + return true; +} + +inline bool get_cert_validity(cert_t cert, time_t ¬_before, + time_t ¬_after) { + if (!cert) return false; + auto x509 = static_cast(cert); + + auto nb = X509_get0_notBefore(x509); + auto na = X509_get0_notAfter(x509); + if (!nb || !na) return false; + + ASN1_TIME *epoch = ASN1_TIME_new(); + if (!epoch) return false; + auto se = detail::scope_exit([&] { ASN1_TIME_free(epoch); }); + + if (!ASN1_TIME_set(epoch, 0)) return false; + + int pday, psec; + + if (!ASN1_TIME_diff(&pday, &psec, epoch, nb)) return false; + not_before = 86400 * (time_t)pday + psec; + + if (!ASN1_TIME_diff(&pday, &psec, epoch, na)) return false; + not_after = 86400 * (time_t)pday + psec; + + return true; +} + +inline std::string get_cert_serial(cert_t cert) { + if (!cert) return ""; + auto x509 = static_cast(cert); + + auto serial = X509_get_serialNumber(x509); + if (!serial) return ""; + + auto bn = ASN1_INTEGER_to_BN(serial, nullptr); + if (!bn) return ""; + + auto hex = BN_bn2hex(bn); + BN_free(bn); + if (!hex) return ""; + + std::string result(hex); + OPENSSL_free(hex); + return result; +} + +inline bool get_cert_der(cert_t cert, std::vector &der) { + if (!cert) return false; + auto x509 = static_cast(cert); + auto len = i2d_X509(x509, nullptr); + if (len < 0) return false; + der.resize(static_cast(len)); + auto p = der.data(); + i2d_X509(x509, &p); + return true; +} + +inline const char *get_sni(const_session_t session) { + if (!session) return nullptr; + auto ssl = static_cast(const_cast(session)); + return SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); +} + +inline uint64_t peek_error() { return ERR_peek_last_error(); } + +inline uint64_t get_error() { return ERR_get_error(); } + +inline std::string error_string(uint64_t code) { + char buf[256]; + ERR_error_string_n(static_cast(code), buf, sizeof(buf)); + return std::string(buf); +} + +inline ca_store_t create_ca_store(const char *pem, size_t len) { + auto mem = BIO_new_mem_buf(pem, static_cast(len)); + if (!mem) { return nullptr; } + auto mem_guard = detail::scope_exit([&] { BIO_free_all(mem); }); + + auto inf = PEM_X509_INFO_read_bio(mem, nullptr, nullptr, nullptr); + if (!inf) { return nullptr; } + + auto store = X509_STORE_new(); + if (store) { + for (auto i = 0; i < static_cast(sk_X509_INFO_num(inf)); i++) { + auto itmp = sk_X509_INFO_value(inf, i); + if (!itmp) { continue; } + if (itmp->x509) { X509_STORE_add_cert(store, itmp->x509); } + if (itmp->crl) { X509_STORE_add_crl(store, itmp->crl); } + } + } + + sk_X509_INFO_pop_free(inf, X509_INFO_free); + return static_cast(store); +} + +inline void free_ca_store(ca_store_t store) { + if (store) { X509_STORE_free(static_cast(store)); } +} + +inline bool set_ca_store(ctx_t ctx, ca_store_t store) { + if (!ctx || !store) { return false; } + auto ssl_ctx = static_cast(ctx); + auto x509_store = static_cast(store); + + // Check if same store is already set + if (SSL_CTX_get_cert_store(ssl_ctx) == x509_store) { return true; } + + // SSL_CTX_set_cert_store takes ownership and frees the old store + SSL_CTX_set_cert_store(ssl_ctx, x509_store); + return true; +} + +inline size_t get_ca_certs(ctx_t ctx, std::vector &certs) { + certs.clear(); + if (!ctx) { return 0; } + auto ssl_ctx = static_cast(ctx); + + auto store = SSL_CTX_get_cert_store(ssl_ctx); + if (!store) { return 0; } + + auto objs = impl::get_store_objects(store); + if (!objs) { return 0; } + auto se = detail::scope_exit([&] { impl::release_store_objects(objs); }); + + auto count = sk_X509_OBJECT_num(objs); + for (decltype(count) i = 0; i < count; i++) { + auto obj = sk_X509_OBJECT_value(objs, i); + if (!obj) { continue; } + if (X509_OBJECT_get_type(obj) == X509_LU_X509) { + auto x509 = X509_OBJECT_get0_X509(obj); + if (x509) { + // Increment reference count so caller can free it + X509_up_ref(x509); + certs.push_back(static_cast(x509)); + } + } + } + return certs.size(); +} + +inline std::vector get_ca_names(ctx_t ctx) { + std::vector names; + if (!ctx) { return names; } + auto ssl_ctx = static_cast(ctx); + + auto store = SSL_CTX_get_cert_store(ssl_ctx); + if (!store) { return names; } + + auto objs = impl::get_store_objects(store); + if (!objs) { return names; } + auto se = detail::scope_exit([&] { impl::release_store_objects(objs); }); + + auto count = sk_X509_OBJECT_num(objs); + for (decltype(count) i = 0; i < count; i++) { + auto obj = sk_X509_OBJECT_value(objs, i); + if (!obj) { continue; } + if (X509_OBJECT_get_type(obj) == X509_LU_X509) { + auto x509 = X509_OBJECT_get0_X509(obj); + if (x509) { + auto subject = X509_get_subject_name(x509); + if (subject) { + char buf[512]; + X509_NAME_oneline(subject, buf, sizeof(buf)); + names.push_back(buf); + } + } + } + } + return names; +} + +inline bool update_server_cert(ctx_t ctx, const char *cert_pem, + const char *key_pem, const char *password) { + if (!ctx || !cert_pem || !key_pem) { return false; } + auto ssl_ctx = static_cast(ctx); + + // Load certificate from PEM + auto cert_bio = BIO_new_mem_buf(cert_pem, -1); + if (!cert_bio) { return false; } + auto cert = PEM_read_bio_X509(cert_bio, nullptr, nullptr, nullptr); + BIO_free(cert_bio); + if (!cert) { return false; } + + // Load private key from PEM + auto key_bio = BIO_new_mem_buf(key_pem, -1); + if (!key_bio) { + X509_free(cert); + return false; + } + auto key = PEM_read_bio_PrivateKey(key_bio, nullptr, nullptr, + password ? const_cast(password) + : nullptr); + BIO_free(key_bio); + if (!key) { + X509_free(cert); + return false; + } + + // Update certificate and key + auto ret = SSL_CTX_use_certificate(ssl_ctx, cert) == 1 && + SSL_CTX_use_PrivateKey(ssl_ctx, key) == 1; + + X509_free(cert); + EVP_PKEY_free(key); + return ret; +} + +inline bool update_server_client_ca(ctx_t ctx, const char *ca_pem) { + if (!ctx || !ca_pem) { return false; } + auto ssl_ctx = static_cast(ctx); + + // Create new X509_STORE from PEM + auto store = create_ca_store(ca_pem, strlen(ca_pem)); + if (!store) { return false; } + + // SSL_CTX_set_cert_store takes ownership + SSL_CTX_set_cert_store(ssl_ctx, static_cast(store)); + + // Set client CA list for client certificate request + auto ca_list = impl::create_client_ca_list_from_pem(ca_pem); + if (ca_list) { + // SSL_CTX_set_client_CA_list takes ownership of ca_list + SSL_CTX_set_client_CA_list(ssl_ctx, ca_list); + } + + return true; +} + +inline bool set_verify_callback(ctx_t ctx, VerifyCallback callback) { + if (!ctx) { return false; } + auto ssl_ctx = static_cast(ctx); + + impl::get_verify_callback() = std::move(callback); + + if (impl::get_verify_callback()) { + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, impl::openssl_verify_callback); + } else { + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, nullptr); + } + return true; +} + +inline long get_verify_error(const_session_t session) { + if (!session) { return -1; } + auto ssl = static_cast(const_cast(session)); + return SSL_get_verify_result(ssl); +} + +inline std::string verify_error_string(long error_code) { + if (error_code == X509_V_OK) { return ""; } + const char *str = X509_verify_cert_error_string(static_cast(error_code)); + return str ? str : "unknown error"; +} + +} // namespace tls + +#endif // CPPHTTPLIB_OPENSSL_SUPPORT + +/* + * Group 9: TLS abstraction layer - Mbed TLS backend + */ + +/* + * Mbed TLS Backend Implementation + */ + +#ifdef CPPHTTPLIB_MBEDTLS_SUPPORT +namespace tls { + +namespace impl { + +// Mbed TLS session wrapper +struct MbedTlsSession { + mbedtls_ssl_context ssl; + socket_t sock = INVALID_SOCKET; + std::string hostname; // For client: set via set_sni + std::string sni_hostname; // For server: received from client via SNI callback + + MbedTlsSession() { mbedtls_ssl_init(&ssl); } + + ~MbedTlsSession() { mbedtls_ssl_free(&ssl); } + + MbedTlsSession(const MbedTlsSession &) = delete; + MbedTlsSession &operator=(const MbedTlsSession &) = delete; +}; + +// Thread-local error code accessor for Mbed TLS (since it doesn't have an error +// queue) +inline int &mbedtls_last_error() { + static thread_local int err = 0; + return err; +} + +// Helper to map Mbed TLS error to ErrorCode +inline ErrorCode map_mbedtls_error(int ret, int &out_errno) { + if (ret == 0) { return ErrorCode::Success; } + if (ret == MBEDTLS_ERR_SSL_WANT_READ) { return ErrorCode::WantRead; } + if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) { return ErrorCode::WantWrite; } + if (ret == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) { + return ErrorCode::PeerClosed; + } + if (ret == MBEDTLS_ERR_NET_CONN_RESET || ret == MBEDTLS_ERR_NET_SEND_FAILED || + ret == MBEDTLS_ERR_NET_RECV_FAILED) { + out_errno = errno; + return ErrorCode::SyscallError; + } + if (ret == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED) { + return ErrorCode::CertVerifyFailed; + } + return ErrorCode::Fatal; +} + +// BIO-like send callback for Mbed TLS +inline int mbedtls_net_send_cb(void *ctx, const unsigned char *buf, + size_t len) { + auto sock = *static_cast(ctx); +#ifdef _WIN32 + auto ret = + send(sock, reinterpret_cast(buf), static_cast(len), 0); + if (ret == SOCKET_ERROR) { + int err = WSAGetLastError(); + if (err == WSAEWOULDBLOCK) { return MBEDTLS_ERR_SSL_WANT_WRITE; } + return MBEDTLS_ERR_NET_SEND_FAILED; + } +#else + auto ret = send(sock, buf, len, 0); + if (ret < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return MBEDTLS_ERR_SSL_WANT_WRITE; + } + return MBEDTLS_ERR_NET_SEND_FAILED; + } +#endif + return static_cast(ret); +} + +// BIO-like recv callback for Mbed TLS +inline int mbedtls_net_recv_cb(void *ctx, unsigned char *buf, size_t len) { + auto sock = *static_cast(ctx); +#ifdef _WIN32 + auto ret = + recv(sock, reinterpret_cast(buf), static_cast(len), 0); + if (ret == SOCKET_ERROR) { + int err = WSAGetLastError(); + if (err == WSAEWOULDBLOCK) { return MBEDTLS_ERR_SSL_WANT_READ; } + return MBEDTLS_ERR_NET_RECV_FAILED; + } +#else + auto ret = recv(sock, buf, len, 0); + if (ret < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return MBEDTLS_ERR_SSL_WANT_READ; + } + return MBEDTLS_ERR_NET_RECV_FAILED; + } +#endif + if (ret == 0) { return MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY; } + return static_cast(ret); +} + +// MbedTlsContext constructor/destructor implementations +inline MbedTlsContext::MbedTlsContext() { + mbedtls_ssl_config_init(&conf); + mbedtls_entropy_init(&entropy); + mbedtls_ctr_drbg_init(&ctr_drbg); + mbedtls_x509_crt_init(&ca_chain); + mbedtls_x509_crt_init(&own_cert); + mbedtls_pk_init(&own_key); +} + +inline MbedTlsContext::~MbedTlsContext() { + mbedtls_pk_free(&own_key); + mbedtls_x509_crt_free(&own_cert); + mbedtls_x509_crt_free(&ca_chain); + mbedtls_ctr_drbg_free(&ctr_drbg); + mbedtls_entropy_free(&entropy); + mbedtls_ssl_config_free(&conf); +} + +// Thread-local storage for SNI captured during handshake +// This is needed because the SNI callback doesn't have a way to pass +// session-specific data before the session is fully set up +inline std::string &mbedpending_sni() { + static thread_local std::string sni; + return sni; +} + +// SNI callback for Mbed TLS server to capture client's SNI hostname +inline int mbedtls_sni_callback(void *p_ctx, mbedtls_ssl_context *ssl, + const unsigned char *name, size_t name_len) { + (void)p_ctx; + (void)ssl; + + // Store SNI name in thread-local storage + // It will be retrieved and stored in the session after handshake + if (name && name_len > 0) { + mbedpending_sni().assign(reinterpret_cast(name), name_len); + } else { + mbedpending_sni().clear(); + } + return 0; // Accept any SNI +} + +inline int mbedtls_verify_callback(void *data, mbedtls_x509_crt *crt, + int cert_depth, uint32_t *flags); + +// MbedTLS verify callback wrapper +inline int mbedtls_verify_callback(void *data, mbedtls_x509_crt *crt, + int cert_depth, uint32_t *flags) { + auto &callback = get_verify_callback(); + if (!callback) { return 0; } // Continue with default verification + + // data points to the MbedTlsSession + auto *session = static_cast(data); + + // Build context + VerifyContext verify_ctx; + verify_ctx.session = static_cast(session); + verify_ctx.cert = static_cast(crt); + verify_ctx.depth = cert_depth; + verify_ctx.preverify_ok = (*flags == 0); + verify_ctx.error_code = static_cast(*flags); + + // Convert Mbed TLS flags to error string + static thread_local char error_buf[256]; + if (*flags != 0) { + mbedtls_x509_crt_verify_info(error_buf, sizeof(error_buf), "", *flags); + verify_ctx.error_string = error_buf; + } else { + verify_ctx.error_string = nullptr; + } + + bool accepted = callback(verify_ctx); + + if (accepted) { + *flags = 0; // Clear all error flags + return 0; + } + return MBEDTLS_ERR_X509_CERT_VERIFY_FAILED; +} + +} // namespace impl + +inline ctx_t create_client_context() { + auto ctx = new (std::nothrow) impl::MbedTlsContext(); + if (!ctx) { return nullptr; } + + ctx->is_server = false; + + // Seed the random number generator + const char *pers = "httplib_client"; + int ret = mbedtls_ctr_drbg_seed( + &ctx->ctr_drbg, mbedtls_entropy_func, &ctx->entropy, + reinterpret_cast(pers), strlen(pers)); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + delete ctx; + return nullptr; + } + + // Set up SSL config for client + ret = mbedtls_ssl_config_defaults(&ctx->conf, MBEDTLS_SSL_IS_CLIENT, + MBEDTLS_SSL_TRANSPORT_STREAM, + MBEDTLS_SSL_PRESET_DEFAULT); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + delete ctx; + return nullptr; + } + + // Set random number generator + mbedtls_ssl_conf_rng(&ctx->conf, mbedtls_ctr_drbg_random, &ctx->ctr_drbg); + + // Default: verify peer certificate + mbedtls_ssl_conf_authmode(&ctx->conf, MBEDTLS_SSL_VERIFY_REQUIRED); + + // Set minimum TLS version to 1.2 +#ifdef CPPHTTPLIB_MBEDTLS_V3 + mbedtls_ssl_conf_min_tls_version(&ctx->conf, MBEDTLS_SSL_VERSION_TLS1_2); +#else + mbedtls_ssl_conf_min_version(&ctx->conf, MBEDTLS_SSL_MAJOR_VERSION_3, + MBEDTLS_SSL_MINOR_VERSION_3); +#endif + + return static_cast(ctx); +} + +inline ctx_t create_server_context() { + auto ctx = new (std::nothrow) impl::MbedTlsContext(); + if (!ctx) { return nullptr; } + + ctx->is_server = true; + + // Seed the random number generator + const char *pers = "httplib_server"; + int ret = mbedtls_ctr_drbg_seed( + &ctx->ctr_drbg, mbedtls_entropy_func, &ctx->entropy, + reinterpret_cast(pers), strlen(pers)); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + delete ctx; + return nullptr; + } + + // Set up SSL config for server + ret = mbedtls_ssl_config_defaults(&ctx->conf, MBEDTLS_SSL_IS_SERVER, + MBEDTLS_SSL_TRANSPORT_STREAM, + MBEDTLS_SSL_PRESET_DEFAULT); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + delete ctx; + return nullptr; + } + + // Set random number generator + mbedtls_ssl_conf_rng(&ctx->conf, mbedtls_ctr_drbg_random, &ctx->ctr_drbg); + + // Default: don't verify client + mbedtls_ssl_conf_authmode(&ctx->conf, MBEDTLS_SSL_VERIFY_NONE); + + // Set minimum TLS version to 1.2 +#ifdef CPPHTTPLIB_MBEDTLS_V3 + mbedtls_ssl_conf_min_tls_version(&ctx->conf, MBEDTLS_SSL_VERSION_TLS1_2); +#else + mbedtls_ssl_conf_min_version(&ctx->conf, MBEDTLS_SSL_MAJOR_VERSION_3, + MBEDTLS_SSL_MINOR_VERSION_3); +#endif + + // Set SNI callback to capture client's SNI hostname + mbedtls_ssl_conf_sni(&ctx->conf, impl::mbedtls_sni_callback, nullptr); + + return static_cast(ctx); +} + +inline void free_context(ctx_t ctx) { + if (ctx) { delete static_cast(ctx); } +} + +inline bool set_min_version(ctx_t ctx, Version version) { + if (!ctx) { return false; } + auto mctx = static_cast(ctx); + +#ifdef CPPHTTPLIB_MBEDTLS_V3 + // Mbed TLS 3.x uses mbedtls_ssl_protocol_version enum + mbedtls_ssl_protocol_version min_ver = MBEDTLS_SSL_VERSION_TLS1_2; + if (version >= Version::TLS1_3) { +#if defined(MBEDTLS_SSL_PROTO_TLS1_3) + min_ver = MBEDTLS_SSL_VERSION_TLS1_3; +#endif + } + mbedtls_ssl_conf_min_tls_version(&mctx->conf, min_ver); +#else + // Mbed TLS 2.x uses major/minor version numbers + int major = MBEDTLS_SSL_MAJOR_VERSION_3; + int minor = MBEDTLS_SSL_MINOR_VERSION_3; // TLS 1.2 + if (version >= Version::TLS1_3) { +#if defined(MBEDTLS_SSL_PROTO_TLS1_3) + minor = MBEDTLS_SSL_MINOR_VERSION_4; // TLS 1.3 +#else + minor = MBEDTLS_SSL_MINOR_VERSION_3; // Fall back to TLS 1.2 +#endif + } + mbedtls_ssl_conf_min_version(&mctx->conf, major, minor); +#endif + return true; +} + +inline bool load_ca_pem(ctx_t ctx, const char *pem, size_t len) { + if (!ctx || !pem) { return false; } + auto mctx = static_cast(ctx); + + // mbedtls_x509_crt_parse expects null-terminated string for PEM + // Add null terminator if not present + std::string pem_str(pem, len); + int ret = mbedtls_x509_crt_parse( + &mctx->ca_chain, reinterpret_cast(pem_str.c_str()), + pem_str.size() + 1); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + mbedtls_ssl_conf_ca_chain(&mctx->conf, &mctx->ca_chain, nullptr); + return true; +} + +inline bool load_ca_file(ctx_t ctx, const char *file_path) { + if (!ctx || !file_path) { return false; } + auto mctx = static_cast(ctx); + + int ret = mbedtls_x509_crt_parse_file(&mctx->ca_chain, file_path); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + mbedtls_ssl_conf_ca_chain(&mctx->conf, &mctx->ca_chain, nullptr); + return true; +} + +inline bool load_ca_dir(ctx_t ctx, const char *dir_path) { + if (!ctx || !dir_path) { return false; } + auto mctx = static_cast(ctx); + + int ret = mbedtls_x509_crt_parse_path(&mctx->ca_chain, dir_path); + if (ret < 0) { // Returns number of certs on success, negative on error + impl::mbedtls_last_error() = ret; + return false; + } + + mbedtls_ssl_conf_ca_chain(&mctx->conf, &mctx->ca_chain, nullptr); + return true; +} + +inline bool load_system_certs(ctx_t ctx) { + if (!ctx) { return false; } + auto mctx = static_cast(ctx); + bool loaded = false; + +#ifdef _WIN32 + loaded = impl::enumerate_windows_system_certs( + [&](const unsigned char *data, size_t len) { + return mbedtls_x509_crt_parse_der(&mctx->ca_chain, data, len) == 0; + }); +#elif defined(__APPLE__) && defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) + loaded = impl::enumerate_macos_keychain_certs( + [&](const unsigned char *data, size_t len) { + return mbedtls_x509_crt_parse_der(&mctx->ca_chain, data, len) == 0; + }); +#else + for (auto path = impl::system_ca_paths(); *path; ++path) { + if (mbedtls_x509_crt_parse_file(&mctx->ca_chain, *path) >= 0) { + loaded = true; + break; + } + } + + if (!loaded) { + for (auto dir = impl::system_ca_dirs(); *dir; ++dir) { + if (mbedtls_x509_crt_parse_path(&mctx->ca_chain, *dir) >= 0) { + loaded = true; + break; + } + } + } +#endif + + if (loaded) { + mbedtls_ssl_conf_ca_chain(&mctx->conf, &mctx->ca_chain, nullptr); + } + return loaded; +} + +inline bool set_client_cert_pem(ctx_t ctx, const char *cert, const char *key, + const char *password) { + if (!ctx || !cert || !key) { return false; } + auto mctx = static_cast(ctx); + + // Parse certificate + std::string cert_str(cert); + int ret = mbedtls_x509_crt_parse( + &mctx->own_cert, + reinterpret_cast(cert_str.c_str()), + cert_str.size() + 1); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + // Parse private key + std::string key_str(key); + const unsigned char *pwd = + password ? reinterpret_cast(password) : nullptr; + size_t pwd_len = password ? strlen(password) : 0; + +#ifdef CPPHTTPLIB_MBEDTLS_V3 + ret = mbedtls_pk_parse_key( + &mctx->own_key, reinterpret_cast(key_str.c_str()), + key_str.size() + 1, pwd, pwd_len, mbedtls_ctr_drbg_random, + &mctx->ctr_drbg); +#else + ret = mbedtls_pk_parse_key( + &mctx->own_key, reinterpret_cast(key_str.c_str()), + key_str.size() + 1, pwd, pwd_len); +#endif + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + // Verify that the certificate and private key match +#ifdef CPPHTTPLIB_MBEDTLS_V3 + ret = mbedtls_pk_check_pair(&mctx->own_cert.pk, &mctx->own_key, + mbedtls_ctr_drbg_random, &mctx->ctr_drbg); +#else + ret = mbedtls_pk_check_pair(&mctx->own_cert.pk, &mctx->own_key); +#endif + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + ret = mbedtls_ssl_conf_own_cert(&mctx->conf, &mctx->own_cert, &mctx->own_key); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + return true; +} + +inline bool set_client_cert_file(ctx_t ctx, const char *cert_path, + const char *key_path, const char *password) { + if (!ctx || !cert_path || !key_path) { return false; } + auto mctx = static_cast(ctx); + + // Parse certificate file + int ret = mbedtls_x509_crt_parse_file(&mctx->own_cert, cert_path); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + // Parse private key file +#ifdef CPPHTTPLIB_MBEDTLS_V3 + ret = mbedtls_pk_parse_keyfile(&mctx->own_key, key_path, password, + mbedtls_ctr_drbg_random, &mctx->ctr_drbg); +#else + ret = mbedtls_pk_parse_keyfile(&mctx->own_key, key_path, password); +#endif + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + // Verify that the certificate and private key match +#ifdef CPPHTTPLIB_MBEDTLS_V3 + ret = mbedtls_pk_check_pair(&mctx->own_cert.pk, &mctx->own_key, + mbedtls_ctr_drbg_random, &mctx->ctr_drbg); +#else + ret = mbedtls_pk_check_pair(&mctx->own_cert.pk, &mctx->own_key); +#endif + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + ret = mbedtls_ssl_conf_own_cert(&mctx->conf, &mctx->own_cert, &mctx->own_key); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + return true; +} + +inline void set_verify_client(ctx_t ctx, bool require) { + if (!ctx) { return; } + auto mctx = static_cast(ctx); + mctx->verify_client = require; + if (require) { + mbedtls_ssl_conf_authmode(&mctx->conf, MBEDTLS_SSL_VERIFY_REQUIRED); + } else { + // If a verify callback is set, use OPTIONAL mode to ensure the callback + // is called (matching OpenSSL behavior). Otherwise use NONE. + mbedtls_ssl_conf_authmode(&mctx->conf, mctx->has_verify_callback + ? MBEDTLS_SSL_VERIFY_OPTIONAL + : MBEDTLS_SSL_VERIFY_NONE); + } +} + +inline session_t create_session(ctx_t ctx, socket_t sock) { + if (!ctx || sock == INVALID_SOCKET) { return nullptr; } + auto mctx = static_cast(ctx); + + auto session = new (std::nothrow) impl::MbedTlsSession(); + if (!session) { return nullptr; } + + session->sock = sock; + + int ret = mbedtls_ssl_setup(&session->ssl, &mctx->conf); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + delete session; + return nullptr; + } + + // Explicitly opt out of in-handshake hostname verification by default; + // since Mbed TLS 3.6.4 a client handshake with certificate verification + // fails outright when no hostname was set. set_sni() installs the real + // hostname for DNS hosts; for IP hosts (where SNI must not be set) the + // caller verifies the certificate identity post-handshake via + // verify_hostname(). + mbedtls_ssl_set_hostname(&session->ssl, nullptr); + + // Set BIO callbacks + mbedtls_ssl_set_bio(&session->ssl, &session->sock, impl::mbedtls_net_send_cb, + impl::mbedtls_net_recv_cb, nullptr); + + // Set per-session verify callback with session pointer if callback is + // registered + if (mctx->has_verify_callback) { + mbedtls_ssl_set_verify(&session->ssl, impl::mbedtls_verify_callback, + session); + } + + return static_cast(session); +} + +inline void free_session(session_t session) { + if (session) { delete static_cast(session); } +} + +inline bool set_sni(session_t session, const char *hostname) { + if (!session || !hostname) { return false; } + auto msession = static_cast(session); + + int ret = mbedtls_ssl_set_hostname(&msession->ssl, hostname); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + msession->hostname = hostname; + return true; +} + +inline bool set_hostname(session_t session, const char *hostname) { + // In Mbed TLS, set_hostname also sets up hostname verification + return set_sni(session, hostname); +} + +inline TlsError connect(session_t session) { + TlsError err; + if (!session) { + err.code = ErrorCode::Fatal; + return err; + } + + auto msession = static_cast(session); + int ret = mbedtls_ssl_handshake(&msession->ssl); + + if (ret == 0) { + err.code = ErrorCode::Success; + } else { + err.code = impl::map_mbedtls_error(ret, err.sys_errno); + err.backend_code = static_cast(-ret); + impl::mbedtls_last_error() = ret; + } + + return err; +} + +inline TlsError accept(session_t session) { + // Same as connect for Mbed TLS - handshake works for both client and server + auto result = connect(session); + + // After successful handshake, capture SNI from thread-local storage + if (result.code == ErrorCode::Success && session) { + auto msession = static_cast(session); + msession->sni_hostname = std::move(impl::mbedpending_sni()); + impl::mbedpending_sni().clear(); + } + + return result; +} + +inline bool connect_nonblocking(session_t session, socket_t sock, + time_t timeout_sec, time_t timeout_usec, + TlsError *err) { + if (!session) { + if (err) { err->code = ErrorCode::Fatal; } + return false; + } + + auto msession = static_cast(session); + + // Set socket to non-blocking mode + detail::set_nonblocking(sock, true); + auto cleanup = + detail::scope_exit([&]() { detail::set_nonblocking(sock, false); }); + + int ret; + while ((ret = mbedtls_ssl_handshake(&msession->ssl)) != 0) { + if (ret == MBEDTLS_ERR_SSL_WANT_READ) { + if (detail::select_read(sock, timeout_sec, timeout_usec) > 0) { + continue; + } + } else if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) { + if (detail::select_write(sock, timeout_sec, timeout_usec) > 0) { + continue; + } + } + + // TlsError or timeout + if (err) { + err->code = impl::map_mbedtls_error(ret, err->sys_errno); + err->backend_code = static_cast(-ret); + } + impl::mbedtls_last_error() = ret; + return false; + } + + if (err) { err->code = ErrorCode::Success; } + return true; +} + +inline bool accept_nonblocking(session_t session, socket_t sock, + time_t timeout_sec, time_t timeout_usec, + TlsError *err) { + // Same implementation as connect for Mbed TLS + bool result = + connect_nonblocking(session, sock, timeout_sec, timeout_usec, err); + + // After successful handshake, capture SNI from thread-local storage + if (result && session) { + auto msession = static_cast(session); + msession->sni_hostname = std::move(impl::mbedpending_sni()); + impl::mbedpending_sni().clear(); + } + + return result; +} + +inline ssize_t read(session_t session, void *buf, size_t len, TlsError &err) { + if (!session || !buf) { + err.code = ErrorCode::Fatal; + return -1; + } + + auto msession = static_cast(session); + int ret = + mbedtls_ssl_read(&msession->ssl, static_cast(buf), len); + + if (ret > 0) { + err.code = ErrorCode::Success; + return static_cast(ret); + } + + if (ret == 0) { + err.code = ErrorCode::PeerClosed; + return 0; + } + + err.code = impl::map_mbedtls_error(ret, err.sys_errno); + err.backend_code = static_cast(-ret); + impl::mbedtls_last_error() = ret; + // mbedTLS signals a clean close_notify via a negative error code rather + // than 0; surface it as a clean EOF the way OpenSSL/wolfSSL do. + if (err.code == ErrorCode::PeerClosed) { return 0; } + return -1; +} + +inline ssize_t write(session_t session, const void *buf, size_t len, + TlsError &err) { + if (!session || !buf) { + err.code = ErrorCode::Fatal; + return -1; + } + + auto msession = static_cast(session); + int ret = mbedtls_ssl_write(&msession->ssl, + static_cast(buf), len); + + if (ret > 0) { + err.code = ErrorCode::Success; + return static_cast(ret); + } + + if (ret == 0) { + err.code = ErrorCode::PeerClosed; + return 0; + } + + err.code = impl::map_mbedtls_error(ret, err.sys_errno); + err.backend_code = static_cast(-ret); + impl::mbedtls_last_error() = ret; + return -1; +} + +inline int pending(const_session_t session) { + if (!session) { return 0; } + auto msession = + static_cast(const_cast(session)); + return static_cast(mbedtls_ssl_get_bytes_avail(&msession->ssl)); +} + +inline void shutdown(session_t session, bool graceful) { + if (!session) { return; } + auto msession = static_cast(session); + + if (graceful) { + // Try to send close_notify, but don't block forever + int ret; + int attempts = 0; + while ((ret = mbedtls_ssl_close_notify(&msession->ssl)) != 0 && + attempts < 3) { + if (ret != MBEDTLS_ERR_SSL_WANT_READ && + ret != MBEDTLS_ERR_SSL_WANT_WRITE) { + break; + } + attempts++; + } + } +} + +inline bool is_peer_closed(session_t session, socket_t sock) { + if (!session || sock == INVALID_SOCKET) { return true; } + auto msession = static_cast(session); + + // Check if there's already decrypted data available in the TLS buffer + // If so, the connection is definitely alive + if (mbedtls_ssl_get_bytes_avail(&msession->ssl) > 0) { return false; } + + // Set socket to non-blocking to avoid blocking on read + detail::set_nonblocking(sock, true); + auto cleanup = + detail::scope_exit([&]() { detail::set_nonblocking(sock, false); }); + + // Try a 1-byte read to check connection status + // Note: This will consume the byte if data is available, but for the + // purpose of checking if peer is closed, this should be acceptable + // since we're only called when we expect the connection might be closing + unsigned char buf; + int ret = mbedtls_ssl_read(&msession->ssl, &buf, 1); + + // If we got data or WANT_READ (would block), connection is alive + if (ret > 0 || ret == MBEDTLS_ERR_SSL_WANT_READ) { return false; } + + // If we get a peer close notify or a connection reset, the peer is closed + return ret == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY || + ret == MBEDTLS_ERR_NET_CONN_RESET || ret == 0; +} + +inline cert_t get_peer_cert(const_session_t session) { + if (!session) { return nullptr; } + auto msession = + static_cast(const_cast(session)); + + // Mbed TLS returns a pointer to the internal peer cert chain. + // WARNING: This pointer is only valid while the session is active. + // Do not use the certificate after calling free_session(). + const mbedtls_x509_crt *cert = mbedtls_ssl_get_peer_cert(&msession->ssl); + return const_cast(cert); +} + +inline void free_cert(cert_t cert) { + // Mbed TLS: peer certificate is owned by the SSL context. + // No-op here, but callers should still call this for cross-backend + // portability. + (void)cert; +} + +inline bool verify_hostname(cert_t cert, const char *hostname) { + if (!cert || !hostname) { return false; } + auto mcert = static_cast(cert); + std::string host_str(hostname); + + // Check if hostname is an IP address (IPv4 or IPv6) + unsigned char ip_bytes[16]; + auto ip_len = impl::parse_ip_address(host_str, ip_bytes); + auto is_ip = ip_len > 0; + + // Check Subject Alternative Names (SAN) + // In Mbed TLS 3.x, subject_alt_names contains raw values without ASN.1 tags + // - DNS names: raw string bytes + // - IP addresses: raw IP bytes (4 for IPv4, 16 for IPv6) + const mbedtls_x509_sequence *san = &mcert->subject_alt_names; + while (san != nullptr && san->buf.p != nullptr && san->buf.len > 0) { + const unsigned char *p = san->buf.p; + size_t len = san->buf.len; + + if (is_ip) { + // For an IP host, only a matching iPAddress SAN of the same family + // (4 bytes for IPv4, 16 bytes for IPv6) may authenticate it. + if (len == ip_len && memcmp(p, ip_bytes, ip_len) == 0) { return true; } + } else { + // Check if this SAN is a DNS name (printable ASCII string) + bool is_dns = len > 0; + for (size_t i = 0; i < len && is_dns; i++) { + if (p[i] < 32 || p[i] > 126) { is_dns = false; } + } + if (is_dns) { + std::string san_name(reinterpret_cast(p), len); + if (detail::match_hostname(san_name, host_str)) { return true; } + } + } + san = san->next; + } + + // Fallback: Check Common Name (CN) in subject. Skipped for IP-literal hosts: + // an IP identity is only valid via an iPAddress SAN, never the CN (RFC 9110; + // the OpenSSL backend's X509_check_ip behaves the same way). + if (!is_ip) { + char cn[256]; + int ret = mbedtls_x509_dn_gets(cn, sizeof(cn), &mcert->subject); + if (ret > 0) { + std::string cn_str(cn); + + // Look for "CN=" in the DN string + size_t cn_pos = cn_str.find("CN="); + if (cn_pos != std::string::npos) { + size_t start = cn_pos + 3; + size_t end = cn_str.find(',', start); + std::string cn_value = + cn_str.substr(start, end == std::string::npos ? end : end - start); + + if (detail::match_hostname(cn_value, host_str)) { return true; } + } + } + } + + return false; +} + +inline uint64_t hostname_mismatch_code() { + return static_cast(MBEDTLS_X509_BADCERT_CN_MISMATCH); +} + +inline long get_verify_result(const_session_t session) { + if (!session) { return -1; } + auto msession = + static_cast(const_cast(session)); + uint32_t flags = mbedtls_ssl_get_verify_result(&msession->ssl); + // Return 0 (X509_V_OK equivalent) if verification passed + return flags == 0 ? 0 : static_cast(flags); +} + +inline std::string get_cert_subject_cn(cert_t cert) { + if (!cert) return ""; + auto x509 = static_cast(cert); + + // Find the CN in the subject + const mbedtls_x509_name *name = &x509->subject; + while (name != nullptr) { + if (MBEDTLS_OID_CMP(MBEDTLS_OID_AT_CN, &name->oid) == 0) { + return std::string(reinterpret_cast(name->val.p), + name->val.len); + } + name = name->next; + } + return ""; +} + +inline std::string get_cert_issuer_name(cert_t cert) { + if (!cert) return ""; + auto x509 = static_cast(cert); + + // Build a human-readable issuer name string + char buf[512]; + int ret = mbedtls_x509_dn_gets(buf, sizeof(buf), &x509->issuer); + if (ret < 0) return ""; + return std::string(buf); +} + +inline bool get_cert_sans(cert_t cert, std::vector &sans) { + sans.clear(); + if (!cert) return false; + auto x509 = static_cast(cert); + + // Parse the Subject Alternative Name extension + const mbedtls_x509_sequence *cur = &x509->subject_alt_names; + while (cur != nullptr) { + if (cur->buf.len > 0) { + // Mbed TLS stores SAN as ASN.1 sequences + // The tag byte indicates the type + const unsigned char *p = cur->buf.p; + size_t len = cur->buf.len; + + // First byte is the tag + unsigned char tag = *p; + p++; + len--; + + // Parse length (simple single-byte length assumed) + if (len > 0 && *p < 0x80) { + size_t value_len = *p; + p++; + len--; + + if (value_len <= len) { + SanEntry entry; + // ASN.1 context tags for GeneralName + switch (tag & 0x1F) { + case 2: // dNSName + entry.type = SanType::DNS; + entry.value = + std::string(reinterpret_cast(p), value_len); + break; + case 7: // iPAddress + entry.type = SanType::IP; + if (value_len == 4) { + // IPv4 + char buf[16]; + snprintf(buf, sizeof(buf), "%d.%d.%d.%d", p[0], p[1], p[2], p[3]); + entry.value = buf; + } else if (value_len == 16) { + // IPv6 + char buf[64]; + snprintf(buf, sizeof(buf), + "%02x%02x:%02x%02x:%02x%02x:%02x%02x:" + "%02x%02x:%02x%02x:%02x%02x:%02x%02x", + p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], p[8], + p[9], p[10], p[11], p[12], p[13], p[14], p[15]); + entry.value = buf; + } + break; + case 1: // rfc822Name (email) + entry.type = SanType::EMAIL; + entry.value = + std::string(reinterpret_cast(p), value_len); + break; + case 6: // uniformResourceIdentifier + entry.type = SanType::URI; + entry.value = + std::string(reinterpret_cast(p), value_len); + break; + default: entry.type = SanType::OTHER; break; + } + + if (!entry.value.empty()) { sans.push_back(std::move(entry)); } + } + } + } + cur = cur->next; + } + return true; +} + +inline bool get_cert_validity(cert_t cert, time_t ¬_before, + time_t ¬_after) { + if (!cert) return false; + auto x509 = static_cast(cert); + + // Convert mbedtls_x509_time to time_t + auto to_time_t = [](const mbedtls_x509_time &t) -> time_t { + struct tm tm_time = {}; + tm_time.tm_year = t.year - 1900; + tm_time.tm_mon = t.mon - 1; + tm_time.tm_mday = t.day; + tm_time.tm_hour = t.hour; + tm_time.tm_min = t.min; + tm_time.tm_sec = t.sec; +#ifdef _WIN32 + return _mkgmtime(&tm_time); +#else + return timegm(&tm_time); +#endif + }; + + not_before = to_time_t(x509->valid_from); + not_after = to_time_t(x509->valid_to); + return true; +} + +inline std::string get_cert_serial(cert_t cert) { + if (!cert) return ""; + auto x509 = static_cast(cert); + + // Convert serial number to hex string + std::string result; + result.reserve(x509->serial.len * 2); + for (size_t i = 0; i < x509->serial.len; i++) { + char hex[3]; + snprintf(hex, sizeof(hex), "%02X", x509->serial.p[i]); + result += hex; + } + return result; +} + +inline bool get_cert_der(cert_t cert, std::vector &der) { + if (!cert) return false; + auto crt = static_cast(cert); + if (!crt->raw.p || crt->raw.len == 0) return false; + der.assign(crt->raw.p, crt->raw.p + crt->raw.len); + return true; +} + +inline const char *get_sni(const_session_t session) { + if (!session) return nullptr; + auto msession = static_cast(session); + + // For server: return SNI received from client during handshake + if (!msession->sni_hostname.empty()) { + return msession->sni_hostname.c_str(); + } + + // For client: return the hostname set via set_sni + if (!msession->hostname.empty()) { return msession->hostname.c_str(); } + + return nullptr; +} + +inline uint64_t peek_error() { + // Mbed TLS doesn't have an error queue, return the last error + return static_cast(-impl::mbedtls_last_error()); +} + +inline uint64_t get_error() { + // Mbed TLS doesn't have an error queue, return and clear the last error + uint64_t err = static_cast(-impl::mbedtls_last_error()); + impl::mbedtls_last_error() = 0; + return err; +} + +inline std::string error_string(uint64_t code) { + char buf[256]; + mbedtls_strerror(-static_cast(code), buf, sizeof(buf)); + return std::string(buf); +} + +inline ca_store_t create_ca_store(const char *pem, size_t len) { + auto *ca_chain = new (std::nothrow) mbedtls_x509_crt; + if (!ca_chain) { return nullptr; } + + mbedtls_x509_crt_init(ca_chain); + + // mbedtls_x509_crt_parse expects null-terminated PEM + int ret = mbedtls_x509_crt_parse(ca_chain, + reinterpret_cast(pem), + len + 1); // +1 for null terminator + if (ret != 0) { + // Try without +1 in case PEM is already null-terminated + ret = mbedtls_x509_crt_parse( + ca_chain, reinterpret_cast(pem), len); + if (ret != 0) { + mbedtls_x509_crt_free(ca_chain); + delete ca_chain; + return nullptr; + } + } + + return static_cast(ca_chain); +} + +inline void free_ca_store(ca_store_t store) { + if (store) { + auto *ca_chain = static_cast(store); + mbedtls_x509_crt_free(ca_chain); + delete ca_chain; + } +} + +inline bool set_ca_store(ctx_t ctx, ca_store_t store) { + if (!ctx || !store) { return false; } + auto *mbed_ctx = static_cast(ctx); + auto *ca_chain = static_cast(store); + + // Free existing CA chain + mbedtls_x509_crt_free(&mbed_ctx->ca_chain); + mbedtls_x509_crt_init(&mbed_ctx->ca_chain); + + // Copy the CA chain (deep copy) + // Parse from the raw data of the source cert + mbedtls_x509_crt *src = ca_chain; + while (src != nullptr) { + int ret = mbedtls_x509_crt_parse_der(&mbed_ctx->ca_chain, src->raw.p, + src->raw.len); + if (ret != 0) { + free_ca_store(store); + return false; + } + src = src->next; + } + + // This function takes ownership of the store; the chain was deep-copied + // above, so release the source + free_ca_store(store); + + // Update the SSL config to use the new CA chain + mbedtls_ssl_conf_ca_chain(&mbed_ctx->conf, &mbed_ctx->ca_chain, nullptr); + return true; +} + +inline size_t get_ca_certs(ctx_t ctx, std::vector &certs) { + certs.clear(); + if (!ctx) { return 0; } + auto *mbed_ctx = static_cast(ctx); + + // Iterate through the CA chain + mbedtls_x509_crt *cert = &mbed_ctx->ca_chain; + while (cert != nullptr && cert->raw.len > 0) { + // Create a copy of the certificate for the caller + auto *copy = new mbedtls_x509_crt; + mbedtls_x509_crt_init(copy); + int ret = mbedtls_x509_crt_parse_der(copy, cert->raw.p, cert->raw.len); + if (ret == 0) { + certs.push_back(static_cast(copy)); + } else { + mbedtls_x509_crt_free(copy); + delete copy; + } + cert = cert->next; + } + return certs.size(); +} + +inline std::vector get_ca_names(ctx_t ctx) { + std::vector names; + if (!ctx) { return names; } + auto *mbed_ctx = static_cast(ctx); + + // Iterate through the CA chain + mbedtls_x509_crt *cert = &mbed_ctx->ca_chain; + while (cert != nullptr && cert->raw.len > 0) { + char buf[512]; + int ret = mbedtls_x509_dn_gets(buf, sizeof(buf), &cert->subject); + if (ret > 0) { names.push_back(buf); } + cert = cert->next; + } + return names; +} + +inline bool update_server_cert(ctx_t ctx, const char *cert_pem, + const char *key_pem, const char *password) { + if (!ctx || !cert_pem || !key_pem) { return false; } + auto *mbed_ctx = static_cast(ctx); + + // Free existing certificate and key + mbedtls_x509_crt_free(&mbed_ctx->own_cert); + mbedtls_pk_free(&mbed_ctx->own_key); + mbedtls_x509_crt_init(&mbed_ctx->own_cert); + mbedtls_pk_init(&mbed_ctx->own_key); + + // Parse certificate PEM + int ret = mbedtls_x509_crt_parse( + &mbed_ctx->own_cert, reinterpret_cast(cert_pem), + strlen(cert_pem) + 1); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + // Parse private key PEM +#ifdef CPPHTTPLIB_MBEDTLS_V3 + ret = mbedtls_pk_parse_key( + &mbed_ctx->own_key, reinterpret_cast(key_pem), + strlen(key_pem) + 1, + password ? reinterpret_cast(password) : nullptr, + password ? strlen(password) : 0, mbedtls_ctr_drbg_random, + &mbed_ctx->ctr_drbg); +#else + ret = mbedtls_pk_parse_key( + &mbed_ctx->own_key, reinterpret_cast(key_pem), + strlen(key_pem) + 1, + password ? reinterpret_cast(password) : nullptr, + password ? strlen(password) : 0); +#endif + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + // Configure SSL to use the new certificate and key + ret = mbedtls_ssl_conf_own_cert(&mbed_ctx->conf, &mbed_ctx->own_cert, + &mbed_ctx->own_key); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + return true; +} + +inline bool update_server_client_ca(ctx_t ctx, const char *ca_pem) { + if (!ctx || !ca_pem) { return false; } + auto *mbed_ctx = static_cast(ctx); + + // Free existing CA chain + mbedtls_x509_crt_free(&mbed_ctx->ca_chain); + mbedtls_x509_crt_init(&mbed_ctx->ca_chain); + + // Parse CA PEM + int ret = mbedtls_x509_crt_parse( + &mbed_ctx->ca_chain, reinterpret_cast(ca_pem), + strlen(ca_pem) + 1); + if (ret != 0) { + impl::mbedtls_last_error() = ret; + return false; + } + + // Update SSL config to use new CA chain + mbedtls_ssl_conf_ca_chain(&mbed_ctx->conf, &mbed_ctx->ca_chain, nullptr); + return true; +} + +inline bool set_verify_callback(ctx_t ctx, VerifyCallback callback) { + if (!ctx) { return false; } + auto *mbed_ctx = static_cast(ctx); + + impl::get_verify_callback() = std::move(callback); + mbed_ctx->has_verify_callback = + static_cast(impl::get_verify_callback()); + + if (mbed_ctx->has_verify_callback) { + // Set OPTIONAL mode to ensure callback is called even when verification + // is disabled (matching OpenSSL behavior where SSL_VERIFY_PEER is set) + mbedtls_ssl_conf_authmode(&mbed_ctx->conf, MBEDTLS_SSL_VERIFY_OPTIONAL); + mbedtls_ssl_conf_verify(&mbed_ctx->conf, impl::mbedtls_verify_callback, + nullptr); + } else { + mbedtls_ssl_conf_verify(&mbed_ctx->conf, nullptr, nullptr); + } + return true; +} + +inline long get_verify_error(const_session_t session) { + if (!session) { return -1; } + auto *msession = + static_cast(const_cast(session)); + return static_cast(mbedtls_ssl_get_verify_result(&msession->ssl)); +} + +inline std::string verify_error_string(long error_code) { + if (error_code == 0) { return ""; } + char buf[256]; + mbedtls_x509_crt_verify_info(buf, sizeof(buf), "", + static_cast(error_code)); + // Remove trailing newline if present + std::string result(buf); + while (!result.empty() && (result.back() == '\n' || result.back() == ' ')) { + result.pop_back(); + } + return result; +} + +} // namespace tls + +#endif // CPPHTTPLIB_MBEDTLS_SUPPORT + +/* + * Group 10: TLS abstraction layer - wolfSSL backend + */ + +/* + * wolfSSL Backend Implementation + */ + +#ifdef CPPHTTPLIB_WOLFSSL_SUPPORT +namespace tls { + +namespace impl { + +// wolfSSL session wrapper +struct WolfSSLSession { + WOLFSSL *ssl = nullptr; + socket_t sock = INVALID_SOCKET; + std::string hostname; // For client: set via set_sni + std::string sni_hostname; // For server: received from client via SNI callback + + WolfSSLSession() = default; + + ~WolfSSLSession() { + if (ssl) { wolfSSL_free(ssl); } + } + + WolfSSLSession(const WolfSSLSession &) = delete; + WolfSSLSession &operator=(const WolfSSLSession &) = delete; +}; + +// Thread-local error code accessor for wolfSSL +inline uint64_t &wolfssl_last_error() { + static thread_local uint64_t err = 0; + return err; +} + +// Helper to map wolfSSL error to ErrorCode. +// ssl_error is the value from wolfSSL_get_error(). +// raw_ret is the raw return value from the wolfSSL call (for low-level error). +inline ErrorCode map_wolfssl_error(WOLFSSL *ssl, int ssl_error, + int &out_errno) { + switch (ssl_error) { + case SSL_ERROR_NONE: return ErrorCode::Success; + case SSL_ERROR_WANT_READ: return ErrorCode::WantRead; + case SSL_ERROR_WANT_WRITE: return ErrorCode::WantWrite; + case SSL_ERROR_ZERO_RETURN: return ErrorCode::PeerClosed; + case SSL_ERROR_SYSCALL: out_errno = errno; return ErrorCode::SyscallError; + default: + if (ssl) { + // wolfSSL stores the low-level error code as a negative value. + // DOMAIN_NAME_MISMATCH (-322) indicates hostname verification failure. + int low_err = ssl_error; // wolfSSL_get_error returns the low-level code + if (low_err == DOMAIN_NAME_MISMATCH) { + return ErrorCode::HostnameMismatch; + } + // Check verify result to distinguish cert verification from generic SSL + // errors. + long vr = wolfSSL_get_verify_result(ssl); + if (vr != 0) { return ErrorCode::CertVerifyFailed; } + } + return ErrorCode::Fatal; + } +} + +// WolfSSLContext constructor/destructor implementations +inline WolfSSLContext::WolfSSLContext() { wolfSSL_Init(); } + +inline WolfSSLContext::~WolfSSLContext() { + if (ctx) { wolfSSL_CTX_free(ctx); } +} + +// Thread-local storage for SNI captured during handshake +inline std::string &wolfssl_pending_sni() { + static thread_local std::string sni; + return sni; +} + +// SNI callback for wolfSSL server to capture client's SNI hostname +inline int wolfssl_sni_callback(WOLFSSL *ssl, int *ret, void *exArg) { + (void)ret; + (void)exArg; + + void *name_data = nullptr; + unsigned short name_len = + wolfSSL_SNI_GetRequest(ssl, WOLFSSL_SNI_HOST_NAME, &name_data); + + if (name_data && name_len > 0) { + wolfssl_pending_sni().assign(static_cast(name_data), + name_len); + } else { + wolfssl_pending_sni().clear(); + } + return 0; // Continue regardless +} + +// wolfSSL verify callback wrapper +inline int wolfssl_verify_callback(int preverify_ok, + WOLFSSL_X509_STORE_CTX *x509_ctx) { + auto &callback = get_verify_callback(); + if (!callback) { return preverify_ok; } + + WOLFSSL_X509 *cert = wolfSSL_X509_STORE_CTX_get_current_cert(x509_ctx); + int depth = wolfSSL_X509_STORE_CTX_get_error_depth(x509_ctx); + int err = wolfSSL_X509_STORE_CTX_get_error(x509_ctx); + + // Get the WOLFSSL object from the X509_STORE_CTX + WOLFSSL *ssl = static_cast(wolfSSL_X509_STORE_CTX_get_ex_data( + x509_ctx, wolfSSL_get_ex_data_X509_STORE_CTX_idx())); + + VerifyContext verify_ctx; + verify_ctx.session = static_cast(ssl); + verify_ctx.cert = static_cast(cert); + verify_ctx.depth = depth; + verify_ctx.preverify_ok = (preverify_ok != 0); + verify_ctx.error_code = static_cast(err); + + if (err != 0) { + verify_ctx.error_string = wolfSSL_X509_verify_cert_error_string(err); + } else { + verify_ctx.error_string = nullptr; + } + + bool accepted = callback(verify_ctx); + return accepted ? 1 : 0; +} + +inline void set_wolfssl_password_cb(WOLFSSL_CTX *ctx, const char *password) { + wolfSSL_CTX_set_default_passwd_cb_userdata(ctx, const_cast(password)); + wolfSSL_CTX_set_default_passwd_cb( + ctx, [](char *buf, int size, int /*rwflag*/, void *userdata) -> int { + auto *pwd = static_cast(userdata); + if (!pwd) return 0; + auto len = static_cast(strlen(pwd)); + if (len > size) len = size; + memcpy(buf, pwd, static_cast(len)); + return len; + }); +} + +} // namespace impl + +inline ctx_t create_client_context() { + auto ctx = new (std::nothrow) impl::WolfSSLContext(); + if (!ctx) { return nullptr; } + + ctx->is_server = false; + + WOLFSSL_METHOD *method = wolfTLSv1_2_client_method(); + if (!method) { + delete ctx; + return nullptr; + } + + ctx->ctx = wolfSSL_CTX_new(method); + if (!ctx->ctx) { + delete ctx; + return nullptr; + } + + // Default: verify peer certificate + wolfSSL_CTX_set_verify(ctx->ctx, SSL_VERIFY_PEER, nullptr); + + return static_cast(ctx); +} + +inline ctx_t create_server_context() { + auto ctx = new (std::nothrow) impl::WolfSSLContext(); + if (!ctx) { return nullptr; } + + ctx->is_server = true; + + WOLFSSL_METHOD *method = wolfTLSv1_2_server_method(); + if (!method) { + delete ctx; + return nullptr; + } + + ctx->ctx = wolfSSL_CTX_new(method); + if (!ctx->ctx) { + delete ctx; + return nullptr; + } + + // Default: don't verify client + wolfSSL_CTX_set_verify(ctx->ctx, SSL_VERIFY_NONE, nullptr); + + // Enable SNI on server + wolfSSL_CTX_SNI_SetOptions(ctx->ctx, WOLFSSL_SNI_HOST_NAME, + WOLFSSL_SNI_CONTINUE_ON_MISMATCH); + wolfSSL_CTX_set_servername_callback(ctx->ctx, impl::wolfssl_sni_callback); + + return static_cast(ctx); +} + +inline void free_context(ctx_t ctx) { + if (ctx) { delete static_cast(ctx); } +} + +inline bool set_min_version(ctx_t ctx, Version version) { + if (!ctx) { return false; } + auto wctx = static_cast(ctx); + + int min_ver = WOLFSSL_TLSV1_2; + if (version >= Version::TLS1_3) { min_ver = WOLFSSL_TLSV1_3; } + + return wolfSSL_CTX_SetMinVersion(wctx->ctx, min_ver) == WOLFSSL_SUCCESS; +} + +inline bool load_ca_pem(ctx_t ctx, const char *pem, size_t len) { + if (!ctx || !pem) { return false; } + auto wctx = static_cast(ctx); + + int ret = wolfSSL_CTX_load_verify_buffer( + wctx->ctx, reinterpret_cast(pem), + static_cast(len), SSL_FILETYPE_PEM); + if (ret != SSL_SUCCESS) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + return false; + } + wctx->ca_pem_data_.append(pem, len); + return true; +} + +inline bool load_ca_file(ctx_t ctx, const char *file_path) { + if (!ctx || !file_path) { return false; } + auto wctx = static_cast(ctx); + + int ret = wolfSSL_CTX_load_verify_locations(wctx->ctx, file_path, nullptr); + if (ret != SSL_SUCCESS) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + return false; + } + return true; +} + +inline bool load_ca_dir(ctx_t ctx, const char *dir_path) { + if (!ctx || !dir_path) { return false; } + auto wctx = static_cast(ctx); + + int ret = wolfSSL_CTX_load_verify_locations(wctx->ctx, nullptr, dir_path); + // wolfSSL may fail if the directory doesn't contain properly hashed certs. + // Unlike OpenSSL which lazily loads certs from directories, wolfSSL scans + // immediately. Return true even on failure since the CA file may have + // already been loaded, matching OpenSSL's lenient behavior. + (void)ret; + return true; +} + +inline bool load_system_certs(ctx_t ctx) { + if (!ctx) { return false; } + auto wctx = static_cast(ctx); + bool loaded = false; + +#ifdef _WIN32 + loaded = impl::enumerate_windows_system_certs( + [&](const unsigned char *data, size_t len) { + return wolfSSL_CTX_load_verify_buffer(wctx->ctx, data, + static_cast(len), + SSL_FILETYPE_ASN1) == SSL_SUCCESS; + }); +#elif defined(__APPLE__) && defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) + loaded = impl::enumerate_macos_keychain_certs( + [&](const unsigned char *data, size_t len) { + return wolfSSL_CTX_load_verify_buffer(wctx->ctx, data, + static_cast(len), + SSL_FILETYPE_ASN1) == SSL_SUCCESS; + }); +#else + for (auto path = impl::system_ca_paths(); *path; ++path) { + if (wolfSSL_CTX_load_verify_locations(wctx->ctx, *path, nullptr) == + SSL_SUCCESS) { + loaded = true; + break; + } + } + + if (!loaded) { + for (auto dir = impl::system_ca_dirs(); *dir; ++dir) { + if (wolfSSL_CTX_load_verify_locations(wctx->ctx, nullptr, *dir) == + SSL_SUCCESS) { + loaded = true; + break; + } + } + } +#endif + + return loaded; +} + +inline bool set_client_cert_pem(ctx_t ctx, const char *cert, const char *key, + const char *password) { + if (!ctx || !cert || !key) { return false; } + auto wctx = static_cast(ctx); + + // Load certificate + int ret = wolfSSL_CTX_use_certificate_buffer( + wctx->ctx, reinterpret_cast(cert), + static_cast(strlen(cert)), SSL_FILETYPE_PEM); + if (ret != SSL_SUCCESS) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + return false; + } + + // Set password callback if password is provided + if (password) { impl::set_wolfssl_password_cb(wctx->ctx, password); } + + // Load private key + ret = wolfSSL_CTX_use_PrivateKey_buffer( + wctx->ctx, reinterpret_cast(key), + static_cast(strlen(key)), SSL_FILETYPE_PEM); + if (ret != SSL_SUCCESS) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + return false; + } + + // Verify that the certificate and private key match + return wolfSSL_CTX_check_private_key(wctx->ctx) == SSL_SUCCESS; +} + +inline bool set_client_cert_file(ctx_t ctx, const char *cert_path, + const char *key_path, const char *password) { + if (!ctx || !cert_path || !key_path) { return false; } + auto wctx = static_cast(ctx); + + // Load certificate file + int ret = + wolfSSL_CTX_use_certificate_file(wctx->ctx, cert_path, SSL_FILETYPE_PEM); + if (ret != SSL_SUCCESS) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + return false; + } + + // Set password callback if password is provided + if (password) { impl::set_wolfssl_password_cb(wctx->ctx, password); } + + // Load private key file + ret = wolfSSL_CTX_use_PrivateKey_file(wctx->ctx, key_path, SSL_FILETYPE_PEM); + if (ret != SSL_SUCCESS) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + return false; + } + + // Verify that the certificate and private key match + return wolfSSL_CTX_check_private_key(wctx->ctx) == SSL_SUCCESS; +} + +inline void set_verify_client(ctx_t ctx, bool require) { + if (!ctx) { return; } + auto wctx = static_cast(ctx); + wctx->verify_client = require; + if (require) { + wolfSSL_CTX_set_verify( + wctx->ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, + wctx->has_verify_callback ? impl::wolfssl_verify_callback : nullptr); + } else { + if (wctx->has_verify_callback) { + wolfSSL_CTX_set_verify(wctx->ctx, SSL_VERIFY_PEER, + impl::wolfssl_verify_callback); + } else { + wolfSSL_CTX_set_verify(wctx->ctx, SSL_VERIFY_NONE, nullptr); + } + } +} + +inline session_t create_session(ctx_t ctx, socket_t sock) { + if (!ctx || sock == INVALID_SOCKET) { return nullptr; } + auto wctx = static_cast(ctx); + + auto session = new (std::nothrow) impl::WolfSSLSession(); + if (!session) { return nullptr; } + + session->sock = sock; + session->ssl = wolfSSL_new(wctx->ctx); + if (!session->ssl) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + delete session; + return nullptr; + } + + wolfSSL_set_fd(session->ssl, static_cast(sock)); + + return static_cast(session); +} + +inline void free_session(session_t session) { + if (session) { delete static_cast(session); } +} + +inline bool set_sni(session_t session, const char *hostname) { + if (!session || !hostname) { return false; } + auto wsession = static_cast(session); + + int ret = wolfSSL_UseSNI(wsession->ssl, WOLFSSL_SNI_HOST_NAME, hostname, + static_cast(strlen(hostname))); + if (ret != WOLFSSL_SUCCESS) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + return false; + } + + // Also set hostname for verification + wolfSSL_check_domain_name(wsession->ssl, hostname); + + wsession->hostname = hostname; + return true; +} + +inline bool set_hostname(session_t session, const char *hostname) { + // In wolfSSL, set_hostname also sets up hostname verification + return set_sni(session, hostname); +} + +inline TlsError connect(session_t session) { + TlsError err; + if (!session) { + err.code = ErrorCode::Fatal; + return err; + } + + auto wsession = static_cast(session); + int ret = wolfSSL_connect(wsession->ssl); + + if (ret == SSL_SUCCESS) { + err.code = ErrorCode::Success; + } else { + int ssl_error = wolfSSL_get_error(wsession->ssl, ret); + err.code = impl::map_wolfssl_error(wsession->ssl, ssl_error, err.sys_errno); + err.backend_code = static_cast(ssl_error); + impl::wolfssl_last_error() = err.backend_code; + } + + return err; +} + +inline TlsError accept(session_t session) { + TlsError err; + if (!session) { + err.code = ErrorCode::Fatal; + return err; + } + + auto wsession = static_cast(session); + int ret = wolfSSL_accept(wsession->ssl); + + if (ret == SSL_SUCCESS) { + err.code = ErrorCode::Success; + // Capture SNI from thread-local storage after successful handshake + wsession->sni_hostname = std::move(impl::wolfssl_pending_sni()); + impl::wolfssl_pending_sni().clear(); + } else { + int ssl_error = wolfSSL_get_error(wsession->ssl, ret); + err.code = impl::map_wolfssl_error(wsession->ssl, ssl_error, err.sys_errno); + err.backend_code = static_cast(ssl_error); + impl::wolfssl_last_error() = err.backend_code; + } + + return err; +} + +inline bool connect_nonblocking(session_t session, socket_t sock, + time_t timeout_sec, time_t timeout_usec, + TlsError *err) { + if (!session) { + if (err) { err->code = ErrorCode::Fatal; } + return false; + } + + auto wsession = static_cast(session); + + // Set socket to non-blocking mode + detail::set_nonblocking(sock, true); + auto cleanup = + detail::scope_exit([&]() { detail::set_nonblocking(sock, false); }); + + int ret; + while ((ret = wolfSSL_connect(wsession->ssl)) != SSL_SUCCESS) { + int ssl_error = wolfSSL_get_error(wsession->ssl, ret); + if (ssl_error == SSL_ERROR_WANT_READ) { + if (detail::select_read(sock, timeout_sec, timeout_usec) > 0) { + continue; + } + } else if (ssl_error == SSL_ERROR_WANT_WRITE) { + if (detail::select_write(sock, timeout_sec, timeout_usec) > 0) { + continue; + } + } + + // Error or timeout + if (err) { + err->code = + impl::map_wolfssl_error(wsession->ssl, ssl_error, err->sys_errno); + err->backend_code = static_cast(ssl_error); + } + impl::wolfssl_last_error() = static_cast(ssl_error); + return false; + } + + if (err) { err->code = ErrorCode::Success; } + return true; +} + +inline bool accept_nonblocking(session_t session, socket_t sock, + time_t timeout_sec, time_t timeout_usec, + TlsError *err) { + if (!session) { + if (err) { err->code = ErrorCode::Fatal; } + return false; + } + + auto wsession = static_cast(session); + + // Set socket to non-blocking mode + detail::set_nonblocking(sock, true); + auto cleanup = + detail::scope_exit([&]() { detail::set_nonblocking(sock, false); }); + + int ret; + while ((ret = wolfSSL_accept(wsession->ssl)) != SSL_SUCCESS) { + int ssl_error = wolfSSL_get_error(wsession->ssl, ret); + if (ssl_error == SSL_ERROR_WANT_READ) { + if (detail::select_read(sock, timeout_sec, timeout_usec) > 0) { + continue; + } + } else if (ssl_error == SSL_ERROR_WANT_WRITE) { + if (detail::select_write(sock, timeout_sec, timeout_usec) > 0) { + continue; + } + } + + // Error or timeout + if (err) { + err->code = + impl::map_wolfssl_error(wsession->ssl, ssl_error, err->sys_errno); + err->backend_code = static_cast(ssl_error); + } + impl::wolfssl_last_error() = static_cast(ssl_error); + return false; + } + + if (err) { err->code = ErrorCode::Success; } + + // Capture SNI from thread-local storage after successful handshake + wsession->sni_hostname = std::move(impl::wolfssl_pending_sni()); + impl::wolfssl_pending_sni().clear(); + + return true; +} + +inline ssize_t read(session_t session, void *buf, size_t len, TlsError &err) { + if (!session || !buf) { + err.code = ErrorCode::Fatal; + return -1; + } + + auto wsession = static_cast(session); + int ret = wolfSSL_read(wsession->ssl, buf, static_cast(len)); + + if (ret > 0) { + err.code = ErrorCode::Success; + return static_cast(ret); + } + + if (ret == 0) { + err.code = ErrorCode::PeerClosed; + return 0; + } + + int ssl_error = wolfSSL_get_error(wsession->ssl, ret); + err.code = impl::map_wolfssl_error(wsession->ssl, ssl_error, err.sys_errno); + err.backend_code = static_cast(ssl_error); + impl::wolfssl_last_error() = err.backend_code; + return -1; +} + +inline ssize_t write(session_t session, const void *buf, size_t len, + TlsError &err) { + if (!session || !buf) { + err.code = ErrorCode::Fatal; + return -1; + } + + auto wsession = static_cast(session); + int ret = wolfSSL_write(wsession->ssl, buf, static_cast(len)); + + if (ret > 0) { + err.code = ErrorCode::Success; + return static_cast(ret); + } + + // wolfSSL_write returns 0 when the peer has sent a close_notify. + // Treat this as an error (return -1) so callers don't spin in a + // write loop adding zero to the offset. + if (ret == 0) { + err.code = ErrorCode::PeerClosed; + return -1; + } + + int ssl_error = wolfSSL_get_error(wsession->ssl, ret); + err.code = impl::map_wolfssl_error(wsession->ssl, ssl_error, err.sys_errno); + err.backend_code = static_cast(ssl_error); + impl::wolfssl_last_error() = err.backend_code; + return -1; +} + +inline int pending(const_session_t session) { + if (!session) { return 0; } + auto wsession = + static_cast(const_cast(session)); + return wolfSSL_pending(wsession->ssl); +} + +inline void shutdown(session_t session, bool graceful) { + if (!session) { return; } + auto wsession = static_cast(session); + + if (graceful) { + int ret; + int attempts = 0; + while ((ret = wolfSSL_shutdown(wsession->ssl)) != SSL_SUCCESS && + attempts < 3) { + int ssl_error = wolfSSL_get_error(wsession->ssl, ret); + if (ssl_error != SSL_ERROR_WANT_READ && + ssl_error != SSL_ERROR_WANT_WRITE) { + break; + } + attempts++; + } + } else { + wolfSSL_shutdown(wsession->ssl); + } +} + +inline bool is_peer_closed(session_t session, socket_t sock) { + if (!session || sock == INVALID_SOCKET) { return true; } + auto wsession = static_cast(session); + + // Check if there's already decrypted data available + if (wolfSSL_pending(wsession->ssl) > 0) { return false; } + + // Set socket to non-blocking to avoid blocking on read + detail::set_nonblocking(sock, true); + auto cleanup = + detail::scope_exit([&]() { detail::set_nonblocking(sock, false); }); + + // Peek 1 byte to check connection status without consuming data + unsigned char buf; + int ret = wolfSSL_peek(wsession->ssl, &buf, 1); + + // If we got data or WANT_READ (would block), connection is alive + if (ret > 0) { return false; } + + int ssl_error = wolfSSL_get_error(wsession->ssl, ret); + if (ssl_error == SSL_ERROR_WANT_READ) { return false; } + + return ssl_error == SSL_ERROR_ZERO_RETURN || ssl_error == SSL_ERROR_SYSCALL || + ret == 0; +} + +inline cert_t get_peer_cert(const_session_t session) { + if (!session) { return nullptr; } + auto wsession = + static_cast(const_cast(session)); + + WOLFSSL_X509 *cert = wolfSSL_get_peer_certificate(wsession->ssl); + return static_cast(cert); +} + +inline void free_cert(cert_t cert) { + if (cert) { wolfSSL_X509_free(static_cast(cert)); } +} + +inline bool verify_hostname(cert_t cert, const char *hostname) { + if (!cert || !hostname) { return false; } + auto x509 = static_cast(cert); + std::string host_str(hostname); + + // Check if hostname is an IP address (IPv4 or IPv6) + unsigned char ip_bytes[16]; + auto ip_len = impl::parse_ip_address(host_str, ip_bytes); + auto is_ip = ip_len > 0; + + // Check Subject Alternative Names + auto *san_names = static_cast( + wolfSSL_X509_get_ext_d2i(x509, NID_subject_alt_name, nullptr, nullptr)); + + if (san_names) { + int san_count = wolfSSL_sk_num(san_names); + for (int i = 0; i < san_count; i++) { + auto *names = + static_cast(wolfSSL_sk_value(san_names, i)); + if (!names) continue; + + if (!is_ip && names->type == WOLFSSL_GEN_DNS) { + // DNS name + unsigned char *dns_name = nullptr; + int dns_len = wolfSSL_ASN1_STRING_to_UTF8(&dns_name, names->d.dNSName); + if (dns_name && dns_len > 0) { + std::string san_name(reinterpret_cast(dns_name), + static_cast(dns_len)); + XFREE(dns_name, nullptr, DYNAMIC_TYPE_OPENSSL); + if (detail::match_hostname(san_name, host_str)) { + wolfSSL_sk_free(san_names); + return true; + } + } + } else if (is_ip && names->type == WOLFSSL_GEN_IPADD) { + // IP address: only an iPAddress SAN of the same family (4 bytes for + // IPv4, 16 bytes for IPv6) may authenticate the host. + unsigned char *ip_data = wolfSSL_ASN1_STRING_data(names->d.iPAddress); + auto san_ip_len = wolfSSL_ASN1_STRING_length(names->d.iPAddress); + if (ip_data && san_ip_len == static_cast(ip_len) && + memcmp(ip_data, ip_bytes, ip_len) == 0) { + wolfSSL_sk_free(san_names); + return true; + } + } + } + wolfSSL_sk_free(san_names); + } + + // Fallback: Check Common Name (CN) in subject. Skipped for IP-literal hosts: + // an IP identity is only valid via an iPAddress SAN, never the CN (RFC 9110; + // the OpenSSL backend's X509_check_ip behaves the same way). + auto subject = is_ip ? nullptr : wolfSSL_X509_get_subject_name(x509); + if (subject) { + char cn[256] = {}; + int cn_len = wolfSSL_X509_NAME_get_text_by_NID(subject, NID_commonName, cn, + sizeof(cn)); + if (cn_len > 0) { + std::string cn_str(cn, static_cast(cn_len)); + if (detail::match_hostname(cn_str, host_str)) { return true; } + } + } + + return false; +} + +inline uint64_t hostname_mismatch_code() { + return static_cast(DOMAIN_NAME_MISMATCH); +} + +inline long get_verify_result(const_session_t session) { + if (!session) { return -1; } + auto wsession = + static_cast(const_cast(session)); + long result = wolfSSL_get_verify_result(wsession->ssl); + return result; +} + +inline std::string get_cert_subject_cn(cert_t cert) { + if (!cert) return ""; + auto x509 = static_cast(cert); + + WOLFSSL_X509_NAME *subject = wolfSSL_X509_get_subject_name(x509); + if (!subject) return ""; + + char cn[256] = {}; + int cn_len = wolfSSL_X509_NAME_get_text_by_NID(subject, NID_commonName, cn, + sizeof(cn)); + if (cn_len <= 0) return ""; + return std::string(cn, static_cast(cn_len)); +} + +inline std::string get_cert_issuer_name(cert_t cert) { + if (!cert) return ""; + auto x509 = static_cast(cert); + + WOLFSSL_X509_NAME *issuer = wolfSSL_X509_get_issuer_name(x509); + if (!issuer) return ""; + + char *name_str = wolfSSL_X509_NAME_oneline(issuer, nullptr, 0); + if (!name_str) return ""; + + std::string result(name_str); + XFREE(name_str, nullptr, DYNAMIC_TYPE_OPENSSL); + return result; +} + +inline bool get_cert_sans(cert_t cert, std::vector &sans) { + sans.clear(); + if (!cert) return false; + auto x509 = static_cast(cert); + + auto *san_names = static_cast( + wolfSSL_X509_get_ext_d2i(x509, NID_subject_alt_name, nullptr, nullptr)); + if (!san_names) return true; // No SANs is not an error + + int count = wolfSSL_sk_num(san_names); + for (int i = 0; i < count; i++) { + auto *name = + static_cast(wolfSSL_sk_value(san_names, i)); + if (!name) continue; + + SanEntry entry; + switch (name->type) { + case WOLFSSL_GEN_DNS: { + entry.type = SanType::DNS; + unsigned char *dns_name = nullptr; + int dns_len = wolfSSL_ASN1_STRING_to_UTF8(&dns_name, name->d.dNSName); + if (dns_name && dns_len > 0) { + entry.value = std::string(reinterpret_cast(dns_name), + static_cast(dns_len)); + XFREE(dns_name, nullptr, DYNAMIC_TYPE_OPENSSL); + } + break; + } + case WOLFSSL_GEN_IPADD: { + entry.type = SanType::IP; + unsigned char *ip_data = wolfSSL_ASN1_STRING_data(name->d.iPAddress); + int ip_len = wolfSSL_ASN1_STRING_length(name->d.iPAddress); + if (ip_data && ip_len == 4) { + char buf[16]; + snprintf(buf, sizeof(buf), "%d.%d.%d.%d", ip_data[0], ip_data[1], + ip_data[2], ip_data[3]); + entry.value = buf; + } else if (ip_data && ip_len == 16) { + char buf[64]; + snprintf(buf, sizeof(buf), + "%02x%02x:%02x%02x:%02x%02x:%02x%02x:" + "%02x%02x:%02x%02x:%02x%02x:%02x%02x", + ip_data[0], ip_data[1], ip_data[2], ip_data[3], ip_data[4], + ip_data[5], ip_data[6], ip_data[7], ip_data[8], ip_data[9], + ip_data[10], ip_data[11], ip_data[12], ip_data[13], + ip_data[14], ip_data[15]); + entry.value = buf; + } + break; + } + case WOLFSSL_GEN_EMAIL: + entry.type = SanType::EMAIL; + { + unsigned char *email = nullptr; + int email_len = wolfSSL_ASN1_STRING_to_UTF8(&email, name->d.rfc822Name); + if (email && email_len > 0) { + entry.value = std::string(reinterpret_cast(email), + static_cast(email_len)); + XFREE(email, nullptr, DYNAMIC_TYPE_OPENSSL); + } + } + break; + case WOLFSSL_GEN_URI: + entry.type = SanType::URI; + { + unsigned char *uri = nullptr; + int uri_len = wolfSSL_ASN1_STRING_to_UTF8( + &uri, name->d.uniformResourceIdentifier); + if (uri && uri_len > 0) { + entry.value = std::string(reinterpret_cast(uri), + static_cast(uri_len)); + XFREE(uri, nullptr, DYNAMIC_TYPE_OPENSSL); + } + } + break; + default: entry.type = SanType::OTHER; break; + } + + if (!entry.value.empty()) { sans.push_back(std::move(entry)); } + } + wolfSSL_sk_free(san_names); + return true; +} + +inline bool get_cert_validity(cert_t cert, time_t ¬_before, + time_t ¬_after) { + if (!cert) return false; + auto x509 = static_cast(cert); + + const WOLFSSL_ASN1_TIME *nb = wolfSSL_X509_get_notBefore(x509); + const WOLFSSL_ASN1_TIME *na = wolfSSL_X509_get_notAfter(x509); + + if (!nb || !na) return false; + + // wolfSSL_ASN1_TIME_to_tm is available + struct tm tm_nb = {}, tm_na = {}; + if (wolfSSL_ASN1_TIME_to_tm(nb, &tm_nb) != WOLFSSL_SUCCESS) return false; + if (wolfSSL_ASN1_TIME_to_tm(na, &tm_na) != WOLFSSL_SUCCESS) return false; + +#ifdef _WIN32 + not_before = _mkgmtime(&tm_nb); + not_after = _mkgmtime(&tm_na); +#else + not_before = timegm(&tm_nb); + not_after = timegm(&tm_na); +#endif + return true; +} + +inline std::string get_cert_serial(cert_t cert) { + if (!cert) return ""; + auto x509 = static_cast(cert); + + WOLFSSL_ASN1_INTEGER *serial_asn1 = wolfSSL_X509_get_serialNumber(x509); + if (!serial_asn1) return ""; + + // Get the serial number data + int len = serial_asn1->length; + unsigned char *data = serial_asn1->data; + if (!data || len <= 0) return ""; + + std::string result; + result.reserve(static_cast(len) * 2); + for (int i = 0; i < len; i++) { + char hex[3]; + snprintf(hex, sizeof(hex), "%02X", data[i]); + result += hex; + } + return result; +} + +inline bool get_cert_der(cert_t cert, std::vector &der) { + if (!cert) return false; + auto x509 = static_cast(cert); + + int der_len = 0; + const unsigned char *der_data = wolfSSL_X509_get_der(x509, &der_len); + if (!der_data || der_len <= 0) return false; + + der.assign(der_data, der_data + der_len); + return true; +} + +inline const char *get_sni(const_session_t session) { + if (!session) return nullptr; + auto wsession = static_cast(session); + + // For server: return SNI received from client during handshake + if (!wsession->sni_hostname.empty()) { + return wsession->sni_hostname.c_str(); + } + + // For client: return the hostname set via set_sni + if (!wsession->hostname.empty()) { return wsession->hostname.c_str(); } + + return nullptr; +} + +inline uint64_t peek_error() { + return static_cast(wolfSSL_ERR_peek_last_error()); +} + +inline uint64_t get_error() { + uint64_t err = impl::wolfssl_last_error(); + impl::wolfssl_last_error() = 0; + return err; +} + +inline std::string error_string(uint64_t code) { + char buf[256]; + wolfSSL_ERR_error_string(static_cast(code), buf); + return std::string(buf); +} + +inline ca_store_t create_ca_store(const char *pem, size_t len) { + if (!pem || len == 0) { return nullptr; } + // Validate by attempting to load into a temporary ctx + WOLFSSL_CTX *tmp_ctx = wolfSSL_CTX_new(wolfTLSv1_2_client_method()); + if (!tmp_ctx) { return nullptr; } + int ret = wolfSSL_CTX_load_verify_buffer( + tmp_ctx, reinterpret_cast(pem), + static_cast(len), SSL_FILETYPE_PEM); + wolfSSL_CTX_free(tmp_ctx); + if (ret != SSL_SUCCESS) { return nullptr; } + return static_cast( + new impl::WolfSSLCAStore{std::string(pem, len)}); +} + +inline void free_ca_store(ca_store_t store) { + delete static_cast(store); +} + +inline bool set_ca_store(ctx_t ctx, ca_store_t store) { + if (!ctx || !store) { return false; } + auto *wctx = static_cast(ctx); + auto *ca = static_cast(store); + int ret = wolfSSL_CTX_load_verify_buffer( + wctx->ctx, reinterpret_cast(ca->pem_data.data()), + static_cast(ca->pem_data.size()), SSL_FILETYPE_PEM); + if (ret == SSL_SUCCESS) { wctx->ca_pem_data_ += ca->pem_data; } + // This function takes ownership of the store; the PEM data was copied into + // the context, so release the source + free_ca_store(store); + return ret == SSL_SUCCESS; +} + +inline size_t get_ca_certs(ctx_t ctx, std::vector &certs) { + certs.clear(); + if (!ctx) { return 0; } + auto *wctx = static_cast(ctx); + if (wctx->ca_pem_data_.empty()) { return 0; } + + const std::string &pem = wctx->ca_pem_data_; + const std::string begin_marker = "-----BEGIN CERTIFICATE-----"; + const std::string end_marker = "-----END CERTIFICATE-----"; + size_t pos = 0; + while ((pos = pem.find(begin_marker, pos)) != std::string::npos) { + size_t end_pos = pem.find(end_marker, pos); + if (end_pos == std::string::npos) { break; } + end_pos += end_marker.size(); + std::string cert_pem = pem.substr(pos, end_pos - pos); + WOLFSSL_X509 *x509 = wolfSSL_X509_load_certificate_buffer( + reinterpret_cast(cert_pem.data()), + static_cast(cert_pem.size()), WOLFSSL_FILETYPE_PEM); + if (x509) { certs.push_back(static_cast(x509)); } + pos = end_pos; + } + return certs.size(); +} + +inline std::vector get_ca_names(ctx_t ctx) { + std::vector names; + if (!ctx) { return names; } + auto *wctx = static_cast(ctx); + if (wctx->ca_pem_data_.empty()) { return names; } + + const std::string &pem = wctx->ca_pem_data_; + const std::string begin_marker = "-----BEGIN CERTIFICATE-----"; + const std::string end_marker = "-----END CERTIFICATE-----"; + size_t pos = 0; + while ((pos = pem.find(begin_marker, pos)) != std::string::npos) { + size_t end_pos = pem.find(end_marker, pos); + if (end_pos == std::string::npos) { break; } + end_pos += end_marker.size(); + std::string cert_pem = pem.substr(pos, end_pos - pos); + WOLFSSL_X509 *x509 = wolfSSL_X509_load_certificate_buffer( + reinterpret_cast(cert_pem.data()), + static_cast(cert_pem.size()), WOLFSSL_FILETYPE_PEM); + if (x509) { + WOLFSSL_X509_NAME *subject = wolfSSL_X509_get_subject_name(x509); + if (subject) { + char *name_str = wolfSSL_X509_NAME_oneline(subject, nullptr, 0); + if (name_str) { + names.push_back(name_str); + XFREE(name_str, nullptr, DYNAMIC_TYPE_OPENSSL); + } + } + wolfSSL_X509_free(x509); + } + pos = end_pos; + } + return names; +} + +inline bool update_server_cert(ctx_t ctx, const char *cert_pem, + const char *key_pem, const char *password) { + if (!ctx || !cert_pem || !key_pem) { return false; } + auto *wctx = static_cast(ctx); + + // Load new certificate + int ret = wolfSSL_CTX_use_certificate_buffer( + wctx->ctx, reinterpret_cast(cert_pem), + static_cast(strlen(cert_pem)), SSL_FILETYPE_PEM); + if (ret != SSL_SUCCESS) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + return false; + } + + // Set password if provided + if (password) { impl::set_wolfssl_password_cb(wctx->ctx, password); } + + // Load new private key + ret = wolfSSL_CTX_use_PrivateKey_buffer( + wctx->ctx, reinterpret_cast(key_pem), + static_cast(strlen(key_pem)), SSL_FILETYPE_PEM); + if (ret != SSL_SUCCESS) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + return false; + } + + return true; +} + +inline bool update_server_client_ca(ctx_t ctx, const char *ca_pem) { + if (!ctx || !ca_pem) { return false; } + auto *wctx = static_cast(ctx); + + int ret = wolfSSL_CTX_load_verify_buffer( + wctx->ctx, reinterpret_cast(ca_pem), + static_cast(strlen(ca_pem)), SSL_FILETYPE_PEM); + if (ret != SSL_SUCCESS) { + impl::wolfssl_last_error() = + static_cast(wolfSSL_ERR_peek_last_error()); + return false; + } + return true; +} + +inline bool set_verify_callback(ctx_t ctx, VerifyCallback callback) { + if (!ctx) { return false; } + auto *wctx = static_cast(ctx); + + impl::get_verify_callback() = std::move(callback); + wctx->has_verify_callback = static_cast(impl::get_verify_callback()); + + if (wctx->has_verify_callback) { + wolfSSL_CTX_set_verify(wctx->ctx, SSL_VERIFY_PEER, + impl::wolfssl_verify_callback); + } else { + wolfSSL_CTX_set_verify( + wctx->ctx, + wctx->verify_client + ? (SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT) + : SSL_VERIFY_NONE, + nullptr); + } + return true; +} + +inline long get_verify_error(const_session_t session) { + if (!session) { return -1; } + auto *wsession = + static_cast(const_cast(session)); + return wolfSSL_get_verify_result(wsession->ssl); +} + +inline std::string verify_error_string(long error_code) { + if (error_code == 0) { return ""; } + const char *str = + wolfSSL_X509_verify_cert_error_string(static_cast(error_code)); + return str ? std::string(str) : std::string(); +} + +} // namespace tls + +#endif // CPPHTTPLIB_WOLFSSL_SUPPORT + +// WebSocket implementation +namespace ws { + +inline bool WebSocket::send_frame(Opcode op, const char *data, size_t len, + bool fin) { + std::lock_guard lock(write_mutex_); + if (closed_) { return false; } + return detail::write_websocket_frame(strm_, op, data, len, fin, !is_server_); +} + +inline ReadResult WebSocket::read(std::string &msg) { + while (!closed_) { + Opcode opcode; + std::string payload; + bool fin; + + if (!impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_, + CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) { + closed_ = true; + return Fail; + } + + switch (opcode) { + case Opcode::Ping: { + std::lock_guard lock(write_mutex_); + detail::write_websocket_frame(strm_, Opcode::Pong, payload.data(), + payload.size(), true, !is_server_); + continue; + } + case Opcode::Pong: { + std::lock_guard lock(ping_mutex_); + unacked_pings_ = 0; + continue; + } + case Opcode::Close: { + if (!closed_.exchange(true)) { + // Echo close frame back + std::lock_guard lock(write_mutex_); + detail::write_websocket_frame(strm_, Opcode::Close, payload.data(), + payload.size(), true, !is_server_); + } + return Fail; + } + case Opcode::Text: + case Opcode::Binary: { + auto result = opcode == Opcode::Text ? Text : Binary; + msg = std::move(payload); + + // Handle fragmentation + if (!fin) { + while (true) { + Opcode cont_opcode; + std::string cont_payload; + bool cont_fin; + if (!impl::read_websocket_frame( + strm_, cont_opcode, cont_payload, cont_fin, is_server_, + CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) { + closed_ = true; + return Fail; + } + if (cont_opcode == Opcode::Ping) { + std::lock_guard lock(write_mutex_); + detail::write_websocket_frame( + strm_, Opcode::Pong, cont_payload.data(), cont_payload.size(), + true, !is_server_); + continue; + } + if (cont_opcode == Opcode::Pong) { + std::lock_guard lock(ping_mutex_); + unacked_pings_ = 0; + continue; + } + if (cont_opcode == Opcode::Close) { + if (!closed_.exchange(true)) { + std::lock_guard lock(write_mutex_); + detail::write_websocket_frame( + strm_, Opcode::Close, cont_payload.data(), + cont_payload.size(), true, !is_server_); + } + return Fail; + } + // RFC 6455: continuation frames must use opcode 0x0 + if (cont_opcode != Opcode::Continuation) { + closed_ = true; + return Fail; + } + msg += cont_payload; + if (msg.size() > CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH) { + closed_ = true; + return Fail; + } + if (cont_fin) { break; } + } + } + // RFC 6455 Section 5.6: text frames must contain valid UTF-8 + if (result == Text && !impl::is_valid_utf8(msg)) { + close(CloseStatus::InvalidPayload, "invalid UTF-8"); + return Fail; + } + return result; + } + default: closed_ = true; return Fail; + } + } + return Fail; +} + +inline bool WebSocket::send(const std::string &data) { + return send_frame(Opcode::Text, data.data(), data.size()); +} + +inline bool WebSocket::send(const char *data, size_t len) { + return send_frame(Opcode::Binary, data, len); +} + +inline void WebSocket::close(CloseStatus status, const std::string &reason) { + if (closed_.exchange(true)) { return; } + ping_cv_.notify_all(); + std::string payload; + auto code = static_cast(status); + payload.push_back(static_cast((code >> 8) & 0xFF)); + payload.push_back(static_cast(code & 0xFF)); + // RFC 6455 Section 5.5: control frame payload must not exceed 125 bytes + // Close frame has 2-byte status code, so reason is limited to 123 bytes + payload += reason.substr(0, 123); + { + std::lock_guard lock(write_mutex_); + detail::write_websocket_frame(strm_, Opcode::Close, payload.data(), + payload.size(), true, !is_server_); + } + + // RFC 6455 Section 7.1.1: after sending a Close frame, wait for the peer's + // Close response before closing the TCP connection. Use a short timeout to + // avoid hanging if the peer doesn't respond. + strm_.set_read_timeout(CPPHTTPLIB_WEBSOCKET_CLOSE_TIMEOUT_SECOND, 0); + Opcode op; + std::string resp; + bool fin; + while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125)) { + if (op == Opcode::Close) { break; } + } +} + +inline WebSocket::~WebSocket() { + { + std::lock_guard lock(ping_mutex_); + closed_ = true; + } + ping_cv_.notify_all(); + if (ping_thread_.joinable()) { ping_thread_.join(); } +} + +inline void WebSocket::start_heartbeat() { + if (ping_interval_sec_ == 0) { return; } + ping_thread_ = std::thread([this]() { + std::unique_lock lock(ping_mutex_); + while (!closed_) { + ping_cv_.wait_for(lock, std::chrono::seconds(ping_interval_sec_)); + if (closed_) { break; } + // If the peer has failed to respond to the previous pings, give up. + // RFC 6455 does not define a pong-timeout mechanism; this is an + // opt-in liveness check controlled by max_missed_pongs_. + if (max_missed_pongs_ > 0 && unacked_pings_ >= max_missed_pongs_) { + lock.unlock(); + close(CloseStatus::GoingAway, "pong timeout"); + return; + } + lock.unlock(); + if (!send_frame(Opcode::Ping, nullptr, 0)) { + lock.lock(); + closed_ = true; + break; + } + lock.lock(); + unacked_pings_++; + } + }); +} + +inline const Request &WebSocket::request() const { return req_; } + +inline bool WebSocket::is_open() const { return !closed_; } + +// WebSocketClient implementation +inline WebSocketClient::WebSocketClient( + const std::string &scheme_host_port_path, const Headers &headers) + : headers_(headers) { + detail::UrlComponents uc; + if (detail::parse_url(scheme_host_port_path, uc) && !uc.scheme.empty() && + !uc.host.empty() && !uc.path.empty()) { + auto &scheme = uc.scheme; + +#ifdef CPPHTTPLIB_SSL_ENABLED + if (scheme != "ws" && scheme != "wss") { +#else + if (scheme != "ws") { +#endif +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + std::string msg = "'" + scheme + "' scheme is not supported."; + throw std::invalid_argument(msg); +#endif + return; + } + + auto is_ssl = scheme == "wss"; + + host_ = std::move(uc.host); + + port_ = is_ssl ? 443 : 80; + if (!uc.port.empty() && !detail::parse_port(uc.port, port_)) { return; } + + path_ = std::move(uc.path); + if (!uc.query.empty()) { path_ += uc.query; } + +#ifdef CPPHTTPLIB_SSL_ENABLED + is_ssl_ = is_ssl; + if (is_ssl_) { + // The context lives as long as the client so that CA configuration + // survives reconnects; sessions are created per connection. + tls_ctx_ = tls::create_client_context(); + if (!tls_ctx_) { return; } + } +#else + if (is_ssl) { return; } +#endif + + is_valid_ = true; + } +} + +inline WebSocketClient::~WebSocketClient() { + shutdown_and_close(); +#ifdef CPPHTTPLIB_SSL_ENABLED + if (tls_ctx_) { + tls::free_context(tls_ctx_); + tls_ctx_ = nullptr; + } +#endif +} + +inline bool WebSocketClient::is_valid() const { return is_valid_; } + +inline void WebSocketClient::shutdown_and_close() { +#ifdef CPPHTTPLIB_SSL_ENABLED + if (is_ssl_) { + if (tls_session_) { + tls::shutdown(tls_session_, true); + tls::free_session(tls_session_); + tls_session_ = nullptr; + } + } +#endif + if (ws_ && ws_->is_open()) { ws_->close(); } + ws_.reset(); + if (sock_ != INVALID_SOCKET) { + detail::shutdown_socket(sock_); + detail::close_socket(sock_); + sock_ = INVALID_SOCKET; + } +} + +inline bool WebSocketClient::create_stream(std::unique_ptr &strm) { +#ifdef CPPHTTPLIB_SSL_ENABLED + if (is_ssl_) { + if (server_certificate_verification_ && !certs_loaded_) { + uint64_t backend_error = 0; + detail::load_client_ca_config(tls_ctx_, ca_cert_file_path_, std::string(), + custom_ca_loaded_, system_ca_mode_, + backend_error); + certs_loaded_ = true; + } + + if (!detail::setup_client_tls_session(host_, tls_ctx_, tls_session_, sock_, + server_certificate_verification_, + read_timeout_sec_, + read_timeout_usec_)) { + return false; + } + + strm = std::unique_ptr(new detail::SSLSocketStream( + sock_, tls_session_, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_)); + return true; + } +#endif + strm = std::unique_ptr( + new detail::SocketStream(sock_, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_)); + return true; +} + +inline bool WebSocketClient::connect() { + if (!is_valid_) { return false; } + shutdown_and_close(); + + // Check is custom IP specified for host_ + std::string ip; + auto it = addr_map_.find(host_); + if (it != addr_map_.end()) { ip = it->second; } + + Error error; + sock_ = detail::create_client_socket( + host_, ip, port_, address_family_, tcp_nodelay_, ipv6_v6only_, + socket_options_, connection_timeout_sec_, connection_timeout_usec_, + read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, + write_timeout_usec_, interface_, error); + + if (sock_ == INVALID_SOCKET) { return false; } + + std::unique_ptr strm; + if (!create_stream(strm)) { + shutdown_and_close(); + return false; + } + + std::string selected_subprotocol; + if (!detail::perform_websocket_handshake(*strm, host_, port_, path_, headers_, + selected_subprotocol)) { + shutdown_and_close(); + return false; + } + subprotocol_ = std::move(selected_subprotocol); + + Request req; + req.method = "GET"; + req.path = path_; + ws_ = std::unique_ptr(new WebSocket(std::move(strm), req, false, + websocket_ping_interval_sec_, + websocket_max_missed_pongs_)); + return true; +} + +inline ReadResult WebSocketClient::read(std::string &msg) { + if (!ws_) { return Fail; } + return ws_->read(msg); +} + +inline bool WebSocketClient::send(const std::string &data) { + if (!ws_) { return false; } + return ws_->send(data); +} + +inline bool WebSocketClient::send(const char *data, size_t len) { + if (!ws_) { return false; } + return ws_->send(data, len); +} + +inline void WebSocketClient::close(CloseStatus status, + const std::string &reason) { + if (ws_) { ws_->close(status, reason); } +} + +inline bool WebSocketClient::is_open() const { return ws_ && ws_->is_open(); } + +inline const std::string &WebSocketClient::subprotocol() const { + return subprotocol_; +} + +inline void WebSocketClient::set_read_timeout(time_t sec, time_t usec) { + read_timeout_sec_ = sec; + read_timeout_usec_ = usec; +} + +inline void WebSocketClient::set_write_timeout(time_t sec, time_t usec) { + write_timeout_sec_ = sec; + write_timeout_usec_ = usec; +} + +inline void WebSocketClient::set_websocket_ping_interval(time_t sec) { + websocket_ping_interval_sec_ = sec; +} + +inline void WebSocketClient::set_websocket_max_missed_pongs(int count) { + websocket_max_missed_pongs_ = count; +} + +inline void WebSocketClient::set_tcp_nodelay(bool on) { tcp_nodelay_ = on; } + +inline void WebSocketClient::set_address_family(int family) { + address_family_ = family; +} + +inline void WebSocketClient::set_ipv6_v6only(bool on) { ipv6_v6only_ = on; } + +inline void WebSocketClient::set_socket_options(SocketOptions socket_options) { + socket_options_ = std::move(socket_options); +} + +inline void WebSocketClient::set_connection_timeout(time_t sec, time_t usec) { + connection_timeout_sec_ = sec; + connection_timeout_usec_ = usec; +} + +inline void WebSocketClient::set_interface(const std::string &intf) { + interface_ = intf; +} + +inline void WebSocketClient::set_hostname_addr_map( + std::map addr_map) { + addr_map_ = std::move(addr_map); +} + +#ifdef CPPHTTPLIB_SSL_ENABLED + +inline void WebSocketClient::set_ca_cert_path(const std::string &path) { + ca_cert_file_path_ = path; +} + +inline void WebSocketClient::set_ca_cert_store(tls::ca_store_t store) { + if (store && tls_ctx_) { + // set_ca_store takes ownership of store + tls::set_ca_store(tls_ctx_, store); + custom_ca_loaded_ = true; + } else if (store) { + tls::free_ca_store(store); + } +} + +inline void WebSocketClient::load_ca_cert_store(const char *ca_cert, + std::size_t size) { + if (tls_ctx_ && ca_cert && size > 0) { + tls::load_ca_pem(tls_ctx_, ca_cert, size); + custom_ca_loaded_ = true; + } +} + +inline void +WebSocketClient::enable_server_certificate_verification(bool enabled) { + server_certificate_verification_ = enabled; +} + +inline void WebSocketClient::enable_system_ca(bool enabled) { + system_ca_mode_ = enabled ? SystemCAMode::Enabled : SystemCAMode::Disabled; +} + +#endif // CPPHTTPLIB_SSL_ENABLED + +} // namespace ws + +// ---------------------------------------------------------------------------- + +} // namespace httplib + +#endif // CPPHTTPLIB_HTTPLIB_H diff --git a/deploy/ocr/vendor/lw.PPOCR.C/examples/http-server/www/index.html b/deploy/ocr/vendor/lw.PPOCR.C/examples/http-server/www/index.html new file mode 100644 index 00000000..b19f108f --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/examples/http-server/www/index.html @@ -0,0 +1,51 @@ + + + + + + lw.PPOCR.C HTTP OCR Demo + + + +

lw.PPOCR.C · HTTP OCR Demo

+
+
图片预览
+ +
+ + + + diff --git a/deploy/ocr/vendor/lw.PPOCR.C/include/lw_infer.h b/deploy/ocr/vendor/lw.PPOCR.C/include/lw_infer.h new file mode 100644 index 00000000..1c21bffb --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/include/lw_infer.h @@ -0,0 +1,381 @@ +#ifndef LW_INFER_H +#define LW_INFER_H + +/* + * Public C ABI for model loading and PP-OCR inference. + * + * Keep this header usable from both C and C++. Public option/info/result + * structures start with struct_size so a caller and a newer library can evolve + * layouts safely. Stable options, output metadata, and result structures copy + * only the common prefix advertised by struct_size. Array element records + * lw_detection_box and lw_ocr_line + * intentionally omit struct_size because their capacities are carried by the + * surrounding result structures. Handles are opaque, and caller-provided buffers + * never cross an allocator boundary; this is especially important for DLL users + * on Windows. + */ + +#include + +#if defined(_WIN32) && defined(LW_PPOCR_C_SHARED) +# if defined(LW_PPOCR_C_BUILD) +# define LW_API __declspec(dllexport) +# else +# define LW_API __declspec(dllimport) +# endif +#elif defined(LW_PPOCR_C_SHARED) && defined(__GNUC__) && __GNUC__ >= 4 +# define LW_API __attribute__((visibility("default"))) +#else +# define LW_API +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define LW_ERROR_MESSAGE_CAPACITY 256u +#define LW_MAX_DIMS 8u + +typedef enum lw_status { + LW_STATUS_OK = 0, + LW_STATUS_INVALID_ARGUMENT = 1, + LW_STATUS_IO_ERROR = 2, + LW_STATUS_OUT_OF_MEMORY = 3, + LW_STATUS_INVALID_FORMAT = 4, + LW_STATUS_UNSUPPORTED_VERSION = 5, + LW_STATUS_OUT_OF_BOUNDS = 6, + LW_STATUS_CHECKSUM_MISMATCH = 7, + LW_STATUS_UNSUPPORTED = 8, + LW_STATUS_INVALID_SHAPE = 9, + LW_STATUS_MEMORY_LIMIT = 10 +} lw_status; + +typedef enum lw_dtype { + LW_DTYPE_F32 = 1, + LW_DTYPE_I32 = 2, + LW_DTYPE_I64 = 3, + LW_DTYPE_U8 = 4 +} lw_dtype; + +/* Output ordering policy. Horizontal LTR is the legacy default. */ +typedef enum lw_reading_order { + LW_READING_ORDER_HORIZONTAL_LTR = 0, + LW_READING_ORDER_VERTICAL_RTL = 1, + LW_READING_ORDER_VERTICAL_LTR = 2 +} lw_reading_order; + +/* Functions return a stable status code. The optional error object adds a + * human-readable diagnostic and must be initialized before the first call. */ +typedef struct lw_error { + uint32_t struct_size; + int32_t code; + char message[LW_ERROR_MESSAGE_CAPACITY]; +} lw_error; + +/* Low-level model/session API. A model owns validated file bytes; a session + * borrows that model and owns concrete shapes plus reusable workspace. */ +typedef struct lw_model_options { + uint32_t struct_size; + uint32_t reserved; + uint64_t max_file_size; +} lw_model_options; + +typedef struct lw_model_info { + uint32_t struct_size; + uint16_t format_major; + uint16_t format_minor; + uint32_t tensor_count; + uint32_t node_count; + uint32_t input_count; + uint32_t output_count; + uint64_t file_size; + uint64_t weight_size; + uint64_t workspace_size; + uint64_t content_checksum; +} lw_model_info; + +typedef struct lw_tensor_desc { + uint32_t struct_size; + uint32_t dtype; + uint32_t rank; + uint32_t reserved; + int32_t dimensions[LW_MAX_DIMS]; +} lw_tensor_desc; + +typedef struct lw_session_options { + uint32_t struct_size; + uint32_t reserved; + uint64_t max_workspace_size; + uint64_t max_tensor_size; +} lw_session_options; + +typedef struct lw_session_info { + uint32_t struct_size; + uint32_t tensor_count; + uint32_t input_count; + uint32_t output_count; + uint64_t workspace_size; +} lw_session_info; + +/* REC consumes one cropped text line. target_width selects the model's dynamic + * width, and required_text_capacity includes the trailing NUL byte. */ +typedef struct lw_recognizer_options { + uint32_t struct_size; + uint32_t target_width; + uint32_t reserved0; + uint32_t reserved1; + uint64_t max_model_file_size; + uint64_t max_workspace_size; + uint64_t max_tensor_size; + uint64_t max_image_pixels; +} lw_recognizer_options; + +typedef struct lw_recognizer_info { + uint32_t struct_size; + uint32_t target_width; + uint32_t input_height; + uint32_t time_steps; + uint32_t class_count; + uint32_t reserved; + uint64_t max_text_capacity; + uint64_t workspace_size; +} lw_recognizer_info; + +typedef struct lw_recognition_result { + uint32_t struct_size; + uint32_t emitted_count; + float score; + uint32_t resized_width; + uint32_t time_steps; + uint32_t reserved; + uint64_t required_text_capacity; +} lw_recognition_result; + +/* CLS predicts text orientation. orientation_degrees is derived from label and + * is informational; the composed OCR policy decides whether to rotate. */ +typedef struct lw_classifier_options { + uint32_t struct_size; + uint32_t reserved; + uint64_t max_model_file_size; + uint64_t max_workspace_size; + uint64_t max_tensor_size; + uint64_t max_image_pixels; +} lw_classifier_options; + +typedef struct lw_classifier_info { + uint32_t struct_size; + uint32_t input_width; + uint32_t input_height; + uint32_t class_count; + uint64_t workspace_size; +} lw_classifier_info; + +typedef struct lw_classification_result { + uint32_t struct_size; + uint32_t label; + float score; + uint32_t resized_width; + uint32_t orientation_degrees; + uint32_t reserved; +} lw_classification_result; + +/* DET returns clockwise source-image quadrilaterals in reading order. Limits + * are explicit so input size and candidate memory remain bounded. */ +typedef struct lw_detector_options { + uint32_t struct_size; + uint32_t limit_side_length; + uint32_t max_candidates; + uint32_t use_dilation; + float bitmap_threshold; + float box_threshold; + float unclip_ratio; + uint32_t reserved; + uint64_t max_model_file_size; + uint64_t max_workspace_size; + uint64_t max_tensor_size; + uint64_t max_image_pixels; +} lw_detector_options; + +typedef struct lw_detector_info { + uint32_t struct_size; + uint32_t limit_side_length; + uint32_t max_candidates; + uint32_t use_dilation; + float bitmap_threshold; + float box_threshold; + float unclip_ratio; + uint32_t reserved; + uint64_t max_image_pixels; +} lw_detector_info; + +typedef struct lw_detection_box { + float x1; + float y1; + float x2; + float y2; + float x3; + float y3; + float x4; + float y4; + float score; + uint32_t reserved; +} lw_detection_box; + +typedef struct lw_detection_result { + uint32_t struct_size; + uint32_t box_count; + uint32_t required_box_capacity; + uint32_t resized_width; + uint32_t resized_height; + uint32_t reserved; + float width_ratio; + float height_ratio; +} lw_detection_result; + +/* Full OCR composes DET, optional CLS and REC. Text is packed into one UTF-8 + * buffer; each line refers to it by text_offset/text_length, without NUL in + * text_length. Pass both output buffers as NULL with zero capacities to query + * the exact required capacities. */ +typedef struct lw_ocr_options { + uint32_t struct_size; + uint32_t use_direction_classification; + float classifier_threshold; + /* Number of independent CLS/REC workers used after DET. Zero selects the + * platform default (online CPUs capped at 8 on native 64-bit, 1 on x86 + * and WebAssembly). */ + uint32_t worker_count; + uint64_t max_crop_pixels; + lw_detector_options detector; + lw_classifier_options classifier; + lw_recognizer_options recognizer; +} lw_ocr_options; + +typedef struct lw_ocr_info { + uint32_t struct_size; + uint32_t use_direction_classification; + uint32_t max_line_capacity; + uint32_t worker_count; + uint64_t max_text_capacity; + uint64_t max_text_capacity_per_line; + uint64_t max_crop_pixels; +} lw_ocr_info; + +typedef struct lw_ocr_line { + lw_detection_box box; + float recognition_score; + float classification_score; + uint32_t classification_label; + uint32_t applied_rotation_degrees; + uint32_t emitted_count; + uint32_t reserved; + uint64_t text_offset; + uint64_t text_length; +} lw_ocr_line; + +typedef struct lw_ocr_result { + uint32_t struct_size; + uint32_t line_count; + uint32_t required_line_capacity; + uint32_t detected_count; + uint32_t detector_resized_width; + uint32_t detector_resized_height; + uint32_t reserved0; + uint32_t reserved1; + uint64_t required_text_capacity; +} lw_ocr_result; + +typedef struct lw_model lw_model; +typedef struct lw_session lw_session; +typedef struct lw_recognizer lw_recognizer; +typedef struct lw_classifier lw_classifier; +typedef struct lw_detector lw_detector; +typedef struct lw_ocr lw_ocr; + +/* Always initialize options/info/result structures through these helpers. + * This sets struct_size and keeps reserved fields at zero for ABI evolution. */ +LW_API void lw_model_options_init(lw_model_options* options); +LW_API void lw_model_info_init(lw_model_info* info); +LW_API void lw_error_init(lw_error* error); +/* Auto-detect LWM or the supported official PP-OCRv6 ONNX subset by content. + * ONNX is normalized to the existing IR; model_info describes that IR. + * This does not change ABI layout or promise general ONNX operator support. */ +LW_API lw_status lw_model_load(const char* path_utf8, const lw_model_options* options, + lw_model** out_model, lw_error* error); +LW_API void lw_model_free(lw_model* model); +LW_API lw_status lw_model_get_info(const lw_model* model, lw_model_info* info); +LW_API void lw_tensor_desc_init(lw_tensor_desc* tensor); +LW_API void lw_session_options_init(lw_session_options* options); +LW_API void lw_session_info_init(lw_session_info* info); +LW_API lw_status lw_session_create(const lw_model* model, const lw_tensor_desc* inputs, + uint32_t input_count, const lw_session_options* options, + lw_session** out_session, lw_error* error); +LW_API void lw_session_free(lw_session* session); +LW_API lw_status lw_session_get_info(const lw_session* session, lw_session_info* info); +LW_API lw_status lw_session_get_output_desc(const lw_session* session, uint32_t output_index, + lw_tensor_desc* output); +LW_API void lw_recognizer_options_init(lw_recognizer_options* options); +LW_API void lw_recognizer_info_init(lw_recognizer_info* info); +LW_API void lw_recognition_result_init(lw_recognition_result* result); +LW_API lw_status lw_recognizer_create(const char* model_path_utf8, const char* dictionary_path_utf8, + const lw_recognizer_options* options, + lw_recognizer** out_recognizer, lw_error* error); +LW_API void lw_recognizer_free(lw_recognizer* recognizer); +LW_API lw_status lw_recognizer_get_info(const lw_recognizer* recognizer, lw_recognizer_info* info); +LW_API lw_status lw_recognizer_recognize_bgr_u8(lw_recognizer* recognizer, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + char* text_utf8, uint64_t text_capacity, + lw_recognition_result* result, lw_error* error); +LW_API void lw_classifier_options_init(lw_classifier_options* options); +LW_API void lw_classifier_info_init(lw_classifier_info* info); +LW_API void lw_classification_result_init(lw_classification_result* result); +LW_API lw_status lw_classifier_create(const char* model_path_utf8, + const lw_classifier_options* options, + lw_classifier** out_classifier, lw_error* error); +LW_API void lw_classifier_free(lw_classifier* classifier); +LW_API lw_status lw_classifier_get_info(const lw_classifier* classifier, lw_classifier_info* info); +LW_API lw_status lw_classifier_classify_bgr_u8(lw_classifier* classifier, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + lw_classification_result* result, lw_error* error); +LW_API void lw_detector_options_init(lw_detector_options* options); +LW_API void lw_detector_info_init(lw_detector_info* info); +LW_API void lw_detection_result_init(lw_detection_result* result); +LW_API lw_status lw_detector_create(const char* model_path_utf8, const lw_detector_options* options, + lw_detector** out_detector, lw_error* error); +LW_API void lw_detector_free(lw_detector* detector); +LW_API lw_status lw_detector_get_info(const lw_detector* detector, lw_detector_info* info); +LW_API lw_status lw_detector_set_reading_order(lw_detector* detector, uint32_t reading_order, + lw_error* error); +LW_API lw_status lw_detector_get_reading_order(const lw_detector* detector, + uint32_t* reading_order, lw_error* error); +LW_API lw_status lw_detector_detect_bgr_u8(lw_detector* detector, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + lw_detection_box* boxes, uint32_t box_capacity, + lw_detection_result* result, lw_error* error); +LW_API void lw_ocr_options_init(lw_ocr_options* options); +LW_API void lw_ocr_info_init(lw_ocr_info* info); +LW_API void lw_ocr_result_init(lw_ocr_result* result); +LW_API lw_status lw_ocr_create(const char* detector_model_path_utf8, + const char* classifier_model_path_utf8, + const char* recognizer_model_path_utf8, + const char* dictionary_path_utf8, const lw_ocr_options* options, + lw_ocr** out_ocr, lw_error* error); +LW_API void lw_ocr_free(lw_ocr* ocr); +LW_API lw_status lw_ocr_get_info(const lw_ocr* ocr, lw_ocr_info* info); +LW_API lw_status lw_ocr_set_reading_order(lw_ocr* ocr, uint32_t reading_order, lw_error* error); +LW_API lw_status lw_ocr_get_reading_order(const lw_ocr* ocr, uint32_t* reading_order, + lw_error* error); +LW_API lw_status lw_ocr_run_bgr_u8(lw_ocr* ocr, const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, lw_ocr_line* lines, + uint32_t line_capacity, char* text_utf8, uint64_t text_capacity, + lw_ocr_result* result, lw_error* error); +LW_API const char* lw_status_string(lw_status status); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/licenses/PaddleOCR-models-APACHE-2.0.txt b/deploy/ocr/vendor/lw.PPOCR.C/licenses/PaddleOCR-models-APACHE-2.0.txt new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/licenses/PaddleOCR-models-APACHE-2.0.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/deploy/ocr/vendor/lw.PPOCR.C/licenses/cpp-httplib-MIT.txt b/deploy/ocr/vendor/lw.PPOCR.C/licenses/cpp-httplib-MIT.txt new file mode 100644 index 00000000..47c418e0 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/licenses/cpp-httplib-MIT.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 yhirose + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/README.md b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/README.md new file mode 100644 index 00000000..c22f49fd --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/README.md @@ -0,0 +1,15 @@ +# PP-OCRv6 tiny analysis inputs + +Source: `lw.PPOCR.Inference/models/ppocrv6-tiny` in the local PPOCR workspace. + +Copied files: + +- `det.onnx` — SHA-256 `193bab7a04fca699a6c82e6abb5b81bdb28177f0abd4062552b04908dafb19f8` +- `cls.onnx` — SHA-256 `dd8b2b61983d76ab230a58da9e0e0e84956b71c3877f2ce6e438fe22d74d2cf2` +- `rec.onnx` — SHA-256 `9ef676d6ed3c88256a2d92c640c44f25b0c40947e111b14b8be8f594091563e6` +- `ppocr_keys.txt` — SHA-256 `46e1b34ef45684cb46d75ac76d355341fe7f0a2c38d6ee02e63ae6b3878019fc` +- `sample.jpg` — SHA-256 `30c417c9f758a3b62718729f5a944f7d2e10cdd2bde0e8ce6785523ddb68ffe9` +- `model.json` — model-package manifest, schema version 1 + +The hashes are checked by the model-analysis tests so silent asset changes fail +the first development gate. diff --git a/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/cls.onnx b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/cls.onnx new file mode 100644 index 00000000..48d4e992 Binary files /dev/null and b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/cls.onnx differ diff --git a/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/det.onnx b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/det.onnx new file mode 100644 index 00000000..0089a4e1 Binary files /dev/null and b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/det.onnx differ diff --git a/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/model.json b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/model.json new file mode 100644 index 00000000..db5c7c78 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/model.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "family": "PP-OCRv6", + "variant": "tiny", + "language": "zh", + "runtime_status": "primary", + "models": { + "det": "det.onnx", + "cls": "cls.onnx", + "rec": "rec.onnx" + }, + "dictionary": "ppocr_keys.txt", + "checksums": { + "det.onnx": "193bab7a04fca699a6c82e6abb5b81bdb28177f0abd4062552b04908dafb19f8", + "cls.onnx": "dd8b2b61983d76ab230a58da9e0e0e84956b71c3877f2ce6e438fe22d74d2cf2", + "rec.onnx": "9ef676d6ed3c88256a2d92c640c44f25b0c40947e111b14b8be8f594091563e6", + "ppocr_keys.txt": "46e1b34ef45684cb46d75ac76d355341fe7f0a2c38d6ee02e63ae6b3878019fc" + }, + "recommended": { + "det": { + "limit_side": 960, + "bitmap_threshold": 0.3, + "box_threshold": 0.6, + "unclip_ratio": 1.6, + "max_candidates": 1000 + }, + "rec": { + "adaptive_width": true, + "max_width": 960 + } + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/ppocr_keys.txt b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/ppocr_keys.txt new file mode 100644 index 00000000..ef005def --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/ppocr_keys.txt @@ -0,0 +1,6904 @@ +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +: +; +< += +> +? +@ +[ +\ +] +^ +_ +` +{ +| +} +~ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +¢ +£ +¤ +¥ +¦ +§ +¨ +© +ª +« +¬ +® +¯ +° +± +² +³ +´ +µ +¶ +· +¸ +¹ +º +» +¼ +½ +¾ +¿ +À +Á + +à +Ä +Å +Æ +Ç +È +É +Ê +Ë +Ì +Í +Î +Ï +Ð +Ñ +Ò +Ó +Ô +Õ +Ö +× +Ø +Ù +Ú +Û +Ü +Ý +Þ +ß +à +á +â +ã +ä +å +æ +ç +è +é +ê +ë +ì +í +î +ï +ð +ñ +ò +ó +ô +õ +ö +÷ +ø +ù +ú +û +ü +ý +þ +ÿ +Ā +ā +Ă +ă +Ą +ą +Ć +ć +Ĉ +ĉ +Ċ +ċ +Č +č +Ď +ď +Đ +đ +Ē +ē +Ĕ +ĕ +Ė +ė +Ę +ę +Ě +ě +Ĝ +ĝ +Ğ +ğ +Ġ +ġ +Ģ +ģ +Ĥ +ĥ +Ħ +ħ +Ĩ +ĩ +Ī +ī +Ĭ +ĭ +Į +į +İ +ı +IJ +ij +Ĵ +ĵ +Ķ +ķ +ĸ +Ĺ +ĺ +Ļ +ļ +Ľ +ľ +Ŀ +ŀ +Ł +ł +Ń +ń +Ņ +ņ +Ň +ň +ʼn +Ŋ +ŋ +Ō +ō +Ŏ +ŏ +Ő +ő +Œ +œ +Ŕ +ŕ +Ŗ +ŗ +Ř +ř +Ś +ś +Ŝ +ŝ +Ş +ş +Š +š +Ţ +ţ +Ť +ť +Ŧ +ŧ +Ũ +ũ +Ū +ū +Ŭ +ŭ +Ů +ů +Ű +ű +Ų +ų +Ŵ +ŵ +Ŷ +ŷ +Ÿ +Ź +ź +Ż +ż +Ž +ž +ſ +ƒ +Α +Β +Γ +Δ +Ε +Ζ +Η +Θ +Ι +Κ +Λ +Μ +Ν +Ξ +Ο +Π +Ρ +Σ +Τ +Υ +Φ +Χ +Ψ +Ω +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +σ +τ +υ +φ +χ +ψ +ω +ϕ +ϵ +Ạ +ồ +– +— +― +‖ +‘ +’ +“ +” +† +‡ +• +‥ +… +‰ +′ +″ +※ +⁎ +⁰ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁺ +⁻ +⁼ +⁽ +⁾ +ⁿ +₀ +₁ +₂ +₃ +₄ +₅ +₆ +₇ +₈ +₉ +₊ +₋ +₌ +₍ +₎ +ₒ +₠ +₡ +₤ +₧ +₩ +₪ +₫ +€ +₭ +₮ +₱ +₹ +₺ +₽ +₿ +℃ +℉ +ℎ +ℏ +ℑ +℘ +ℜ +™ +℧ +Å +ℵ +ℶ +ℷ +ℸ +ℹ +⅀ +Ⅰ +Ⅱ +Ⅲ +Ⅳ +Ⅴ +Ⅵ +Ⅶ +Ⅷ +Ⅸ +Ⅹ +Ⅺ +Ⅻ +ⅰ +ⅱ +ⅲ +ⅳ +ⅴ +ⅵ +ⅶ +ⅷ +ⅸ +ⅹ +ⅺ +ⅻ +← +↑ +→ +↓ +↔ +↕ +↖ +↗ +↘ +↙ +⇐ +⇑ +⇒ +⇓ +⇔ +⇕ +∀ +∂ +∃ +∅ +∆ +∇ +∈ +∉ +∏ +∑ +− +∓ +∕ +∙ +√ +∝ +∞ +∠ +∥ +∧ +∨ +∩ +∪ +∫ +∬ +∮ +∴ +∵ +∶ +∷ +∼ +∽ +≈ +≌ +≒ +≠ +≡ +≤ +≥ +≦ +≧ +≪ +≫ +≮ +≯ +⊂ +⊃ +⊆ +⊇ +⊊ +⊋ +⊕ +⊗ +⊙ +⊥ +⋅ +① +② +③ +④ +⑤ +⑥ +⑦ +⑧ +⑨ +⑩ +⑪ +⑫ +⑬ +⑭ +⑮ +⑯ +⑰ +⑱ +⑲ +⑳ +■ +□ +▲ +△ +▶ +▼ +▽ +◀ +◆ +◇ +○ +◎ +● +☀ +☁ +☂ +★ +☆ +☎ +☐ +☑ +☒ +☔ +☕ +☠ +♀ +♂ +♠ +♡ +♣ +♤ +♥ +♦ +♻ +⚓ +⚠ +⚡ +✓ +✔ +✕ +✖ +✘ +✚ +✠ +✨ +✳ +✴ +✶ +✽ +❄ +❌ +❎ +❓ +❤ + +、 +。 +々 +〆 +〇 +〈 +〉 +《 +》 +「 +」 +『 +』 +【 +】 +〒 +〔 +〕 +〖 +〗 +〜 +〰 +〵 +〽 +㈱ +㊗ +㊙ +一 +丁 +七 +丄 +万 +丈 +三 +上 +下 +不 +与 +丐 +丑 +专 +且 +丕 +世 +丘 +丙 +业 +丛 +东 +丝 +丞 +両 +丢 +两 +严 +丧 +个 +丫 +中 +丰 +串 +临 +丸 +丹 +为 +主 +丼 +丽 +举 +丿 +乂 +乃 +久 +么 +义 +之 +乌 +乍 +乎 +乏 +乐 +乒 +乓 +乔 +乖 +乘 +乙 +乜 +九 +乞 +也 +习 +乡 +书 +买 +乱 +乳 +乸 +乾 +了 +予 +争 +事 +二 +亍 +于 +亏 +云 +互 +亓 +五 +井 +亘 +亚 +些 +亜 +亞 +亟 +亡 +亢 +交 +亥 +亦 +产 +亨 +亩 +享 +京 +亭 +亮 +亲 +亳 +亵 +人 +亿 +什 +仁 +仃 +仄 +仅 +仆 +仇 +今 +介 +仍 +从 +仑 +仓 +仔 +仕 +他 +仗 +付 +仙 +仝 +仞 +仟 +仡 +代 +令 +以 +仨 +仪 +们 +仰 +仲 +件 +价 +仺 +任 +份 +仿 +企 +伈 +伉 +伊 +伍 +伎 +伏 +伐 +休 +众 +优 +伙 +会 +伞 +伟 +传 +伤 +伦 +伪 +伫 +伯 +估 +伱 +伴 +伶 +伸 +伺 +似 +伽 +佃 +但 +位 +低 +住 +佐 +佑 +体 +何 +佗 +佘 +余 +佚 +佛 +作 +佝 +佞 +佟 +你 +佣 +佤 +佧 +佩 +佬 +佯 +佰 +佳 +佶 +佼 +使 +侃 +侄 +來 +侈 +例 +侍 +侏 +侑 +侗 +供 +依 +侠 +侣 +侥 +侦 +侧 +侨 +侩 +侬 +侮 +侯 +侵 +便 +係 +促 +俄 +俊 +俎 +俏 +俐 +俑 +俗 +俘 +俚 +保 +俞 +俟 +信 +俨 +俩 +俪 +俫 +俬 +俭 +修 +俯 +俱 +俵 +俸 +俺 +俾 +倆 +倉 +個 +倌 +倍 +倏 +們 +倒 +倔 +倘 +候 +倚 +倜 +借 +倡 +倦 +倩 +倪 +倫 +倬 +倭 +债 +值 +倾 +偃 +假 +偈 +偉 +偌 +偎 +偏 +偕 +做 +停 +健 +偶 +偷 +偻 +偽 +偿 +傀 +傅 +傈 +傍 +備 +傢 +傣 +傥 +储 +傩 +催 +傲 +傳 +傷 +傻 +像 +僑 +僚 +僧 +僳 +僵 +價 +僻 +儀 +億 +儋 +儒 +儡 +儥 +優 +儱 +儿 +兀 +允 +元 +兄 +充 +兆 +先 +光 +克 +免 +兑 +兒 +兔 +兖 +党 +兜 +兢 +入 +內 +全 +兩 +八 +公 +六 +兮 +兰 +共 +关 +兴 +兵 +其 +具 +典 +兹 +养 +兼 +兽 +兿 +冀 +内 +冈 +冉 +册 +再 +冏 +冒 +冕 +冗 +写 +冚 +军 +农 +冠 +冢 +冤 +冥 +冬 +冮 +冯 +冰 +冲 +决 +况 +冶 +冷 +冻 +冼 +冽 +净 +凄 +准 +凇 +凈 +凉 +凋 +凌 +凍 +减 +凑 +凖 +凛 +凝 +几 +凡 +凤 +処 +凬 +凭 +凯 +凰 +凱 +凳 +凶 +凸 +凹 +出 +击 +函 +凿 +刀 +刁 +刂 +刃 +分 +切 +刈 +刊 +刍 +刑 +划 +列 +刘 +则 +刚 +创 +初 +删 +判 +刨 +利 +刪 +别 +刮 +到 +制 +刷 +券 +刹 +刺 +刻 +刿 +剁 +剂 +剃 +削 +剌 +前 +剎 +剐 +剑 +剔 +剖 +剡 +剥 +剧 +剩 +剪 +副 +割 +創 +剽 +剿 +劇 +劈 +劉 +劍 +劑 +力 +劝 +办 +功 +加 +务 +劣 +动 +助 +努 +劫 +劭 +励 +劲 +劳 +劵 +劾 +势 +勁 +勃 +勅 +勇 +勉 +勋 +勐 +勑 +勒 +動 +勘 +務 +勝 +募 +勤 +勰 +勺 +勾 +勿 +匀 +包 +匆 +匈 +匋 +匍 +匐 +匕 +化 +北 +匙 +匝 +匠 +匡 +匣 +匪 +匮 +匯 +匹 +区 +医 +匾 +匿 +區 +十 +千 +卅 +升 +午 +卉 +半 +华 +协 +卑 +卒 +卓 +協 +单 +卖 +南 +博 +卜 +卞 +卟 +占 +卡 +卢 +卤 +卦 +卧 +卫 +卯 +印 +危 +即 +却 +卵 +卷 +卸 +卿 +厂 +厄 +厅 +历 +厉 +压 +厌 +厍 +厕 +厘 +厚 +厝 +原 +厢 +厥 +厦 +厨 +厩 +厮 +去 +县 +叁 +参 +參 +又 +叉 +及 +友 +双 +反 +发 +叔 +取 +受 +变 +叙 +叛 +叟 +叠 +叢 +口 +古 +句 +另 +叨 +叩 +只 +叫 +召 +叭 +叮 +可 +台 +叱 +史 +右 +叶 +号 +司 +叹 +叻 +叼 +叽 +吁 +吃 +各 +吆 +合 +吉 +吊 +吋 +同 +名 +后 +吏 +吐 +向 +吒 +吓 +吕 +吖 +吗 +君 +吝 +吞 +吟 +吠 +吡 +否 +吧 +吨 +吩 +含 +听 +吭 +吮 +启 +吱 +吲 +吴 +吵 +吸 +吹 +吻 +吼 +吾 +呀 +呃 +呆 +呈 +告 +呋 +呐 +呓 +呕 +呗 +员 +呙 +呛 +呜 +呢 +呤 +呦 +周 +呱 +味 +呵 +呷 +呸 +呻 +呼 +命 +咀 +咂 +咄 +咅 +咆 +咋 +和 +咎 +咏 +咐 +咒 +咔 +咕 +咖 +咙 +咚 +咛 +咣 +咤 +咦 +咧 +咨 +咩 +咪 +咫 +咬 +咭 +咯 +咱 +咳 +咸 +咻 +咽 +咿 +哀 +品 +哄 +哆 +哇 +哈 +哉 +哌 +响 +哎 +哑 +哒 +哓 +哔 +哗 +哚 +哝 +哞 +哟 +員 +哥 +哦 +哧 +哨 +哩 +哪 +哭 +哮 +哲 +哺 +哼 +哽 +唁 +唆 +唇 +唉 +唏 +唐 +唑 +唔 +唘 +唛 +唠 +唢 +唤 +唧 +唬 +售 +唯 +唰 +唱 +唳 +唷 +唻 +唾 +啃 +啄 +商 +啉 +啊 +問 +啓 +啕 +啖 +啜 +啟 +啡 +啤 +啥 +啦 +啧 +啪 +啫 +啬 +啮 +啰 +啵 +啶 +啸 +啻 +啼 +啾 +喀 +喂 +喃 +善 +喆 +喇 +喉 +喊 +喋 +喏 +喔 +喘 +喙 +喜 +喝 +喧 +喫 +喬 +單 +喰 +喱 +喳 +喵 +喷 +喹 +喻 +喽 +嗅 +嗉 +嗑 +嗒 +嗓 +嗔 +嗖 +嗜 +嗝 +嗞 +嗟 +嗡 +嗣 +嗤 +嗦 +嗨 +嗪 +嗬 +嗮 +嗯 +嗲 +嗳 +嗷 +嗽 +嘀 +嘈 +嘉 +嘌 +嘎 +嘘 +嘛 +嘞 +嘟 +嘢 +嘣 +嘤 +嘧 +嘬 +嘱 +嘲 +嘴 +嘶 +嘹 +嘻 +嘿 +噌 +噎 +噗 +噜 +噢 +噤 +器 +噩 +噪 +噬 +噱 +噶 +噻 +噼 +嚎 +嚏 +嚒 +嚓 +嚞 +嚣 +嚷 +嚼 +囊 +囍 +囔 +囗 +囚 +四 +回 +囟 +因 +囡 +团 +囤 +园 +困 +囱 +围 +囹 +固 +国 +图 +囿 +圃 +圆 +圈 +國 +圍 +園 +圖 +圗 +團 +圜 +土 +圣 +圧 +在 +圩 +圭 +地 +圳 +圹 +场 +圻 +圾 +址 +坂 +均 +坊 +坋 +坍 +坎 +坏 +坐 +坑 +块 +坚 +坛 +坝 +坞 +坟 +坠 +坡 +坤 +坦 +坨 +坩 +坪 +坫 +坭 +坯 +坳 +坶 +坷 +坻 +垂 +垃 +垄 +型 +垌 +垒 +垓 +垚 +垛 +垟 +垠 +垡 +垢 +垣 +垦 +垩 +垫 +垭 +垮 +垸 +埂 +埃 +埋 +埌 +城 +埔 +埕 +埗 +埚 +域 +埠 +埭 +執 +培 +基 +堀 +堂 +堃 +堆 +堇 +堑 +堕 +堡 +堤 +堪 +堰 +報 +場 +堵 +堷 +塊 +塌 +塍 +塑 +塔 +塗 +塘 +塞 +填 +塬 +塾 +境 +墅 +墉 +墊 +墒 +墓 +墙 +增 +墟 +墨 +墩 +壁 +壅 +壇 +壐 +壑 +壓 +壕 +壤 +士 +壬 +壮 +声 +壳 +壴 +壶 +壹 +壽 +夀 +处 +夆 +备 +夋 +夌 +复 +夏 +夔 +夕 +外 +夙 +多 +夜 +够 +夢 +夥 +大 +天 +太 +夫 +夬 +夭 +央 +夯 +失 +头 +夷 +夸 +夹 +夺 +奂 +奄 +奇 +奈 +奉 +奋 +奎 +奏 +契 +奓 +奔 +奕 +奖 +套 +奘 +奚 +奠 +奢 +奥 +奧 +女 +奴 +奶 +奸 +她 +好 +如 +妃 +妄 +妆 +妇 +妈 +妊 +妍 +妒 +妓 +妖 +妗 +妙 +妝 +妞 +妤 +妥 +妨 +妩 +妪 +妫 +妮 +妯 +妲 +妳 +妹 +妻 +妾 +姆 +姊 +始 +姌 +姐 +姑 +姒 +姓 +委 +姗 +姘 +姚 +姜 +姝 +姣 +姥 +姨 +姬 +姹 +姻 +姿 +威 +娃 +娄 +娅 +娆 +娇 +娈 +娉 +娌 +娑 +娓 +娘 +娜 +娟 +娠 +娣 +娥 +娩 +娱 +娲 +娴 +娶 +娼 +娽 +婀 +婆 +婉 +婊 +婕 +婚 +婢 +婦 +婧 +婪 +婴 +婵 +婶 +婷 +婺 +婿 +媄 +媒 +媚 +媛 +媲 +媳 +媽 +媾 +嫁 +嫂 +嫉 +嫌 +嫒 +嫔 +嫖 +嫚 +嫡 +嫣 +嫦 +嫩 +嬉 +嬗 +嬰 +嬴 +嬷 +孃 +子 +孑 +孓 +孔 +孕 +孖 +字 +存 +孙 +孚 +孛 +孜 +孝 +孟 +孢 +季 +孤 +学 +孩 +孪 +孫 +孬 +孰 +孳 +孵 +學 +孺 +孽 +宁 +它 +宄 +宅 +宇 +守 +安 +宋 +完 +宏 +宓 +宕 +宗 +官 +宙 +定 +宛 +宜 +宝 +实 +宠 +审 +客 +宣 +室 +宥 +宦 +宪 +宫 +宰 +害 +宴 +宵 +家 +宸 +容 +宽 +宾 +宿 +寂 +寄 +寅 +密 +寇 +富 +寐 +寒 +寓 +寕 +寝 +寞 +察 +寡 +寤 +寥 +實 +寧 +寨 +寫 +寬 +寮 +寰 +寳 +寵 +寶 +寸 +对 +寺 +寻 +导 +対 +寿 +封 +専 +尃 +射 +将 +將 +專 +尉 +尊 +對 +導 +小 +少 +尓 +尔 +尕 +尖 +尘 +尙 +尚 +尝 +尤 +尧 +尬 +就 +尴 +尷 +尸 +尹 +尺 +尻 +尼 +尽 +尾 +尿 +局 +屁 +层 +居 +屆 +屈 +屉 +届 +屋 +屌 +屎 +屏 +屐 +屑 +展 +属 +屠 +屡 +層 +履 +屯 +山 +屹 +屿 +岁 +岂 +岌 +岐 +岑 +岔 +岖 +岗 +岘 +岙 +岚 +岛 +岢 +岩 +岫 +岭 +岱 +岳 +岷 +岸 +岽 +岿 +峁 +峄 +峋 +峒 +峙 +峡 +峤 +峥 +峦 +峨 +峩 +峪 +峭 +峯 +峰 +島 +峻 +崂 +崃 +崆 +崇 +崋 +崎 +崔 +崖 +崛 +崤 +崧 +崩 +崬 +崭 +崮 +崴 +崽 +嵇 +嵊 +嵋 +嵌 +嵐 +嵘 +嵩 +嵬 +嵯 +嶂 +嶙 +嶪 +嶺 +嶽 +巅 +巍 +川 +州 +巡 +巢 +工 +左 +巧 +巨 +巩 +巫 +差 +巯 +己 +已 +巳 +巴 +巷 +巽 +巾 +币 +市 +布 +帅 +帆 +师 +希 +帏 +帐 +帑 +帕 +帖 +帘 +帚 +帛 +帜 +帝 +帥 +带 +帧 +師 +席 +帮 +帯 +帰 +帶 +帷 +常 +帼 +帽 +幂 +幄 +幅 +幌 +幔 +幕 +幚 +幛 +幡 +幢 +干 +平 +年 +并 +幸 +幹 +幺 +幻 +幼 +幽 +广 +庄 +庆 +庇 +床 +序 +庐 +库 +应 +底 +庖 +店 +庙 +庚 +府 +庞 +废 +庠 +庥 +度 +座 +庫 +庭 +庵 +庶 +康 +庸 +庹 +庾 +廉 +廊 +廓 +廖 +廚 +廠 +廣 +廰 +廳 +延 +廷 +建 +廿 +开 +异 +弃 +弄 +弈 +弊 +弋 +弍 +式 +弑 +弓 +引 +弗 +弘 +弛 +弟 +张 +弥 +弦 +弧 +弩 +弭 +弯 +弱 +張 +強 +弹 +强 +弼 +彈 +彐 +归 +当 +录 +彗 +彝 +形 +彤 +彦 +彧 +彩 +彪 +彬 +彭 +彰 +影 +彳 +彷 +役 +彻 +彼 +往 +征 +径 +待 +徇 +很 +徉 +徊 +律 +後 +徐 +徒 +徕 +得 +徘 +徙 +徜 +御 +徨 +復 +循 +微 +徳 +徵 +德 +徹 +徽 +心 +必 +忆 +忌 +忍 +忏 +忐 +忑 +忒 +忖 +志 +忘 +忙 +応 +忠 +忡 +忧 +快 +忱 +念 +忻 +忽 +忿 +怀 +态 +怂 +怅 +怆 +怎 +怏 +怒 +怔 +怕 +怖 +怜 +思 +怠 +怡 +急 +怦 +性 +怨 +怪 +怯 +总 +恂 +恃 +恋 +恍 +恐 +恒 +恕 +恙 +恢 +恣 +恤 +恨 +恩 +恪 +恬 +恭 +息 +恰 +恳 +恶 +恸 +恹 +恺 +恻 +恼 +恽 +恿 +悄 +悅 +悉 +悌 +悍 +悔 +悖 +悚 +悟 +悠 +患 +悦 +您 +悬 +悯 +悱 +悲 +悴 +悵 +悶 +悸 +悻 +悼 +情 +惆 +惊 +惋 +惑 +惕 +惘 +惚 +惜 +惟 +惠 +惦 +惧 +惨 +惩 +惫 +惬 +惭 +惮 +惯 +惰 +想 +惴 +惶 +惹 +惺 +愁 +愈 +愉 +意 +愕 +愚 +愛 +感 +愣 +愤 +愧 +愫 +愿 +慈 +態 +慌 +慎 +慑 +慕 +慢 +慧 +慨 +慰 +慵 +慶 +慷 +憋 +憎 +憔 +憧 +憨 +憩 +憬 +憶 +憾 +懂 +懈 +應 +懊 +懋 +懒 +懦 +懵 +懷 +懿 +戀 +戈 +戊 +戋 +戌 +戍 +戎 +戏 +成 +我 +戒 +或 +战 +戚 +戛 +戟 +戢 +截 +戬 +戮 +戳 +戴 +戶 +户 +戾 +房 +所 +扁 +扇 +扈 +扉 +手 +才 +扎 +扑 +扒 +打 +扔 +托 +扙 +扛 +扞 +扣 +扦 +执 +扩 +扪 +扫 +扬 +扭 +扮 +扯 +扰 +扳 +扶 +批 +扼 +找 +承 +技 +抄 +抉 +把 +抑 +抒 +抓 +投 +抖 +抗 +折 +抚 +抛 +抠 +抡 +抢 +护 +报 +抨 +披 +抬 +抱 +抵 +抹 +抻 +押 +抽 +抿 +拂 +拄 +担 +拆 +拇 +拈 +拉 +拌 +拍 +拎 +拐 +拒 +拓 +拔 +拖 +拗 +拘 +拙 +拚 +招 +拜 +拟 +拢 +拣 +拥 +拦 +拧 +拨 +择 +括 +拭 +拮 +拯 +拱 +拳 +拴 +拷 +拼 +拽 +拾 +拿 +持 +挂 +指 +按 +挎 +挑 +挖 +挚 +挛 +挝 +挞 +挟 +挠 +挡 +挣 +挤 +挥 +挨 +挪 +挫 +振 +挲 +挺 +挽 +捂 +捅 +捆 +捉 +捋 +捌 +捍 +捎 +捏 +捐 +捕 +捞 +损 +捡 +换 +捣 +捧 +据 +捲 +捶 +捷 +捺 +捻 +掀 +掂 +掇 +授 +掉 +掊 +掌 +掏 +掐 +排 +掖 +掘 +掠 +探 +掣 +接 +控 +推 +掩 +措 +掬 +掰 +掳 +掴 +掷 +掸 +掺 +揉 +揍 +描 +提 +插 +揖 +揚 +換 +握 +揣 +揩 +揪 +揭 +援 +揸 +揽 +搀 +搁 +搂 +搅 +損 +搏 +搐 +搓 +搔 +搜 +搞 +搪 +搬 +搭 +搴 +携 +搽 +摁 +摄 +摆 +摇 +摈 +摊 +摒 +摔 +摘 +摞 +摧 +摩 +摸 +摹 +撂 +撃 +撅 +撇 +撑 +撒 +撕 +撞 +撤 +撩 +撬 +播 +撮 +撰 +撵 +撷 +撸 +撻 +撼 +擀 +擂 +擅 +操 +擎 +擒 +擘 +據 +擞 +擢 +擦 +攀 +攒 +攘 +攝 +攥 +攫 +支 +收 +攸 +改 +攻 +放 +政 +故 +效 +敌 +敏 +救 +敕 +敖 +教 +敛 +敝 +敞 +敟 +敢 +散 +敦 +敬 +数 +敲 +整 +敷 +數 +文 +斋 +斌 +斐 +斑 +斓 +斗 +料 +斛 +斜 +斟 +斡 +斤 +斥 +斧 +斩 +断 +斯 +新 +方 +於 +施 +旁 +旅 +旋 +旌 +旎 +族 +旖 +旗 +无 +既 +日 +旦 +旧 +旨 +早 +旬 +旭 +旮 +旯 +旱 +时 +旷 +旸 +旺 +昀 +昂 +昆 +昇 +昊 +昌 +明 +昏 +易 +昔 +昕 +昙 +昝 +星 +映 +春 +昧 +昨 +昭 +是 +昱 +昴 +昵 +昶 +昼 +显 +晁 +時 +晃 +晋 +晌 +晏 +晒 +晓 +晔 +晕 +晖 +晗 +晚 +晞 +晟 +晤 +晦 +晧 +晨 +普 +景 +晰 +晳 +晴 +晶 +晷 +智 +晾 +暂 +暃 +暄 +暇 +暑 +暖 +暗 +暢 +暧 +暨 +暮 +暴 +暹 +曉 +曙 +曜 +曝 +曦 +曬 +曰 +曲 +曳 +更 +書 +曹 +曼 +曾 +替 +最 +會 +月 +有 +朋 +服 +朐 +朔 +朕 +朗 +望 +朝 +期 +朦 +木 +未 +末 +本 +札 +术 +朱 +朴 +朵 +机 +朽 +朿 +杀 +杂 +权 +杆 +杈 +杉 +李 +杏 +材 +村 +杖 +杜 +杞 +束 +杠 +条 +来 +杨 +杩 +杭 +杯 +杰 +東 +杲 +杳 +杵 +杷 +松 +板 +极 +构 +枇 +枉 +枋 +析 +枕 +林 +枚 +果 +枝 +枞 +枢 +枣 +枪 +枫 +枭 +枯 +枰 +枱 +枳 +架 +枷 +枸 +枼 +柄 +柏 +某 +柑 +柒 +染 +柔 +柘 +柚 +柜 +柞 +柠 +查 +柩 +柬 +柯 +柱 +柳 +柴 +柿 +栀 +栅 +标 +栈 +栉 +栋 +栎 +栏 +树 +栓 +栖 +栗 +校 +栩 +株 +样 +核 +根 +格 +栽 +栾 +桀 +桁 +桂 +桃 +桅 +框 +案 +桉 +桌 +桎 +桐 +桑 +桓 +桔 +桖 +桠 +桡 +桢 +档 +桥 +桦 +桧 +桨 +桩 +桶 +桷 +桼 +梁 +梅 +梆 +梏 +梓 +梗 +條 +梢 +梦 +梧 +梨 +梭 +梯 +械 +梳 +梵 +检 +棉 +棋 +棍 +棒 +棕 +棘 +棚 +棛 +棟 +棠 +棣 +森 +棰 +棱 +棵 +棺 +棽 +椁 +椅 +椋 +植 +椎 +椐 +椒 +椤 +椭 +椰 +椴 +椹 +椽 +椿 +楂 +楊 +楓 +楔 +楗 +楚 +楝 +楞 +楠 +楣 +楦 +業 +極 +楷 +楸 +楹 +楼 +楽 +楿 +概 +榄 +榆 +榈 +榉 +榔 +榕 +榛 +榜 +榞 +榧 +榨 +榫 +榭 +榮 +榴 +榷 +榻 +構 +槌 +槎 +槐 +様 +槛 +槟 +槲 +槽 +槿 +樂 +樊 +樓 +標 +樟 +模 +樣 +樨 +横 +樱 +樵 +樸 +樹 +樽 +樾 +橄 +橇 +橋 +橘 +橙 +機 +橡 +橪 +橱 +橹 +橼 +檀 +檐 +檔 +檗 +檩 +檬 +櫥 +權 +欠 +次 +欢 +欣 +欧 +欲 +欺 +款 +歆 +歇 +歉 +歌 +歐 +歙 +歡 +止 +正 +此 +步 +武 +歧 +歪 +歷 +歸 +歹 +死 +歼 +殁 +殃 +殆 +殇 +殉 +殊 +残 +殒 +殖 +殚 +殡 +殴 +段 +殷 +殺 +殼 +殿 +毁 +毂 +毅 +毋 +母 +每 +毒 +毓 +比 +毕 +毗 +毙 +毛 +毡 +毫 +毯 +毽 +氏 +氐 +民 +氓 +气 +氖 +気 +氘 +氙 +氚 +氛 +氟 +氡 +氢 +氣 +氤 +氦 +氧 +氨 +氩 +氪 +氮 +氯 +氰 +氲 +水 +永 +氽 +汀 +汁 +求 +汇 +汉 +汊 +汐 +汕 +汗 +汛 +汝 +汞 +江 +池 +污 +汤 +汨 +汩 +汪 +汰 +汲 +汴 +汶 +汹 +決 +汽 +汾 +沁 +沂 +沃 +沅 +沈 +沉 +沌 +沏 +沐 +沓 +沔 +沙 +沛 +沟 +没 +沣 +沤 +沥 +沦 +沧 +沪 +沫 +沭 +沮 +沱 +河 +沸 +油 +治 +沼 +沽 +沾 +沿 +泄 +泅 +泉 +泊 +泌 +泓 +泔 +法 +泗 +泛 +泞 +泠 +泡 +波 +泣 +泥 +注 +泩 +泪 +泫 +泮 +泯 +泰 +泱 +泳 +泵 +泷 +泸 +泺 +泻 +泼 +泽 +泾 +洁 +洄 +洋 +洒 +洗 +洙 +洛 +洞 +津 +洪 +洮 +洱 +洲 +洳 +洵 +洺 +活 +洼 +洽 +派 +流 +浃 +浅 +浆 +浇 +浈 +浊 +测 +浍 +济 +浏 +浐 +浑 +浒 +浓 +浔 +浙 +浚 +浜 +浠 +浣 +浥 +浦 +浩 +浪 +浮 +浴 +海 +浸 +浼 +涂 +涅 +涇 +消 +涉 +涌 +涎 +涓 +涔 +涕 +涛 +涝 +涞 +涟 +涠 +涡 +涣 +涤 +润 +涧 +涨 +涩 +涪 +涫 +涮 +涯 +液 +涵 +涸 +涿 +淀 +淄 +淅 +淆 +淇 +淋 +淌 +淑 +淖 +淘 +淙 +淚 +淝 +淞 +淡 +淤 +淦 +淨 +淩 +淫 +淬 +淮 +深 +淳 +混 +淹 +添 +淼 +清 +渊 +渌 +渍 +渎 +渐 +渑 +渔 +渗 +渚 +渝 +渠 +渡 +渣 +渤 +渥 +温 +測 +渭 +港 +渲 +渴 +游 +渺 +湃 +湄 +湍 +湓 +湖 +湘 +湛 +湟 +湫 +湮 +湯 +湾 +湿 +溃 +溅 +溆 +溉 +溏 +源 +溜 +溝 +溟 +溢 +溥 +溧 +溪 +溫 +溯 +溱 +溴 +溶 +溺 +滁 +滂 +滇 +滋 +滏 +滑 +滓 +滔 +滕 +滘 +滙 +滚 +滞 +滟 +满 +滢 +滤 +滥 +滦 +滨 +滩 +滬 +滴 +滿 +漁 +漂 +漆 +漉 +漏 +漓 +演 +漕 +漠 +漢 +漩 +漪 +漫 +漯 +漱 +漳 +漾 +潇 +潋 +潍 +潔 +潘 +潜 +潞 +潢 +潤 +潦 +潭 +潮 +潴 +潸 +潺 +潼 +澄 +澈 +澍 +澎 +澜 +澡 +澤 +澧 +澳 +澶 +澹 +激 +濂 +濃 +濉 +濑 +濒 +濕 +濞 +濟 +濠 +濡 +濤 +濮 +濯 +濱 +瀑 +瀘 +瀚 +瀛 +瀣 +瀵 +瀹 +灌 +灏 +灞 +灡 +灣 +火 +灭 +灯 +灰 +灵 +灶 +灸 +灼 +灾 +灿 +炀 +炅 +炉 +炊 +炎 +炒 +炔 +炕 +炖 +炙 +炜 +炝 +炫 +炬 +炭 +炮 +炯 +炳 +炸 +点 +為 +炼 +炽 +烁 +烂 +烃 +烈 +烊 +烏 +烘 +烙 +烛 +烜 +烟 +烤 +烦 +烧 +烨 +烩 +烫 +烬 +热 +烯 +烷 +烹 +烺 +烽 +焉 +焊 +焕 +焖 +焗 +焘 +焙 +焚 +無 +焦 +焯 +焰 +焱 +然 +煅 +煊 +煌 +煎 +煖 +煙 +煜 +煞 +煤 +煥 +煦 +照 +煨 +煮 +煲 +煸 +煽 +熄 +熊 +熏 +熔 +熘 +熙 +熟 +熠 +熥 +熨 +熬 +熱 +熳 +熵 +熹 +燁 +燃 +燈 +燎 +燒 +燕 +燘 +燙 +燚 +燜 +營 +燥 +燦 +燧 +燮 +燴 +爆 +爍 +爐 +爨 +爪 +爬 +爭 +爯 +爰 +爱 +爲 +爵 +父 +爷 +爸 +爹 +爺 +爻 +爽 +爾 +片 +版 +牌 +牍 +牒 +牙 +牛 +牟 +牡 +牢 +牦 +牧 +物 +牯 +牲 +牵 +特 +牺 +犀 +犁 +犇 +犊 +犍 +犟 +犬 +犯 +状 +犷 +犸 +犹 +狀 +狂 +狄 +狈 +狍 +狐 +狒 +狗 +狙 +狞 +狠 +狡 +狩 +独 +狭 +狮 +狰 +狱 +狸 +狼 +猎 +猕 +猖 +猗 +猛 +猜 +猝 +猥 +猩 +猪 +猫 +猬 +献 +猴 +猷 +猾 +猿 +獅 +獐 +獒 +獗 +獠 +獨 +獭 +獲 +獸 +獻 +玄 +率 +玉 +王 +玑 +玖 +玛 +玟 +玥 +玩 +玫 +玮 +环 +现 +玲 +玳 +玷 +玺 +玻 +珀 +珂 +珈 +珉 +珊 +珍 +珏 +珐 +珑 +珙 +珞 +珠 +珣 +珥 +珩 +班 +珲 +珺 +現 +球 +琅 +理 +琉 +琊 +琏 +琐 +琚 +琛 +琢 +琥 +琦 +琨 +琪 +琬 +琮 +琯 +琰 +琳 +琴 +琵 +琶 +琼 +瑁 +瑄 +瑕 +瑗 +瑙 +瑚 +瑛 +瑜 +瑞 +瑟 +瑩 +瑪 +瑭 +瑰 +瑶 +瑷 +瑾 +璀 +璁 +璃 +璇 +璈 +璋 +璎 +璐 +璜 +璞 +璟 +璧 +璨 +璩 +環 +璺 +瓒 +瓜 +瓠 +瓢 +瓣 +瓤 +瓦 +瓮 +瓯 +瓴 +瓶 +瓷 +瓿 +甄 +甘 +甙 +甚 +甜 +生 +產 +産 +甥 +用 +甩 +甫 +甬 +甭 +甯 +田 +由 +甲 +申 +电 +男 +甸 +甹 +町 +画 +甾 +畅 +畈 +界 +畏 +畐 +畔 +留 +畜 +略 +畦 +番 +畫 +畲 +畴 +畵 +當 +畸 +畹 +畿 +疃 +疆 +疌 +疏 +疑 +疖 +疗 +疙 +疚 +疝 +疟 +疡 +疣 +疤 +疥 +疫 +疮 +疯 +疱 +疲 +疳 +疵 +疸 +疹 +疼 +疽 +疾 +痂 +病 +症 +痈 +痉 +痊 +痒 +痔 +痕 +痘 +痛 +痞 +痠 +痢 +痣 +痤 +痧 +痨 +痪 +痫 +痰 +痱 +痴 +痹 +痼 +痿 +瘀 +瘁 +瘊 +瘘 +瘙 +瘟 +瘠 +瘢 +瘤 +瘦 +瘩 +瘪 +瘫 +瘴 +瘸 +瘾 +瘿 +療 +癌 +癔 +癖 +癜 +癞 +癣 +癫 +癸 +登 +發 +白 +百 +皂 +的 +皆 +皇 +皈 +皋 +皎 +皑 +皓 +皖 +皙 +皮 +皱 +皲 +皴 +皺 +皿 +盂 +盅 +盆 +盈 +益 +盎 +盏 +盐 +监 +盒 +盔 +盖 +盗 +盘 +盛 +盜 +盟 +盤 +盥 +目 +盯 +盱 +盲 +直 +相 +盹 +盼 +盾 +省 +眈 +眉 +看 +眙 +眞 +真 +眠 +眦 +眨 +眩 +眭 +眯 +眶 +眷 +眸 +眺 +眼 +着 +睁 +睇 +睐 +睑 +睛 +睡 +睢 +督 +睦 +睨 +睫 +睬 +睹 +睽 +睾 +睿 +瞄 +瞅 +瞌 +瞎 +瞑 +瞒 +瞓 +瞟 +瞠 +瞥 +瞧 +瞩 +瞪 +瞬 +瞭 +瞰 +瞳 +瞻 +瞿 +矗 +矛 +矜 +矞 +矢 +矣 +知 +矩 +矫 +短 +矮 +石 +矶 +矸 +矽 +矾 +矿 +砀 +码 +砂 +砌 +砍 +砒 +研 +砖 +砚 +砜 +砝 +砣 +砥 +砦 +砧 +砬 +砭 +砰 +破 +砷 +砸 +砺 +砻 +砼 +砾 +础 +硂 +硅 +硒 +硕 +硖 +硚 +硝 +硪 +硫 +硬 +确 +硼 +碁 +碇 +碉 +碌 +碍 +碎 +碑 +碓 +碗 +碘 +碚 +碟 +碣 +碧 +碩 +碰 +碱 +碲 +碳 +碴 +碶 +確 +碼 +碾 +磁 +磅 +磊 +磋 +磐 +磕 +磚 +磨 +磬 +磴 +磷 +磺 +礁 +礴 +示 +礼 +社 +祀 +祁 +祈 +祉 +祎 +祐 +祖 +祗 +祙 +祚 +祛 +祜 +祝 +神 +祟 +祠 +祢 +祥 +票 +祭 +祯 +祷 +祸 +祺 +禀 +禁 +禄 +禅 +福 +禦 +禧 +禮 +禹 +禺 +离 +禽 +禾 +秀 +私 +秃 +秆 +秉 +秋 +种 +科 +秒 +秕 +秘 +租 +秣 +秤 +秦 +秧 +秩 +秭 +积 +称 +秸 +移 +秽 +稀 +程 +稍 +税 +稔 +稗 +稚 +稞 +稠 +稣 +種 +稳 +稷 +稹 +稻 +稼 +稽 +稿 +穂 +穆 +穗 +穩 +穰 +穴 +穵 +究 +穷 +穸 +穹 +空 +穿 +突 +窃 +窄 +窈 +窍 +窑 +窒 +窕 +窖 +窗 +窘 +窜 +窝 +窟 +窠 +窥 +窦 +窨 +窿 +立 +竖 +站 +竞 +竟 +章 +竣 +童 +竭 +端 +競 +竹 +竺 +竽 +竿 +笃 +笆 +笈 +笋 +笏 +笑 +笔 +笕 +笙 +笛 +笠 +符 +笨 +笫 +第 +笺 +笼 +筆 +等 +筋 +筏 +筐 +筑 +筒 +答 +策 +筛 +筝 +筠 +筱 +筵 +筷 +筹 +签 +简 +箍 +箐 +箔 +箕 +算 +管 +箩 +箫 +箬 +箭 +箱 +箴 +箸 +節 +篁 +篆 +篇 +築 +篑 +篓 +篙 +篝 +篡 +篦 +篪 +篮 +篱 +篷 +簇 +簋 +簌 +簡 +簧 +簪 +簸 +簿 +籁 +籍 +籣 +籤 +米 +类 +籼 +籽 +粉 +粑 +粒 +粕 +粗 +粘 +粟 +粢 +粤 +粥 +粧 +粪 +粮 +粱 +粲 +粳 +粵 +粹 +粼 +粽 +精 +粿 +糁 +糅 +糊 +糍 +糕 +糖 +糗 +糙 +糜 +糝 +糟 +糠 +糯 +系 +紀 +約 +紅 +紊 +紋 +納 +純 +紗 +級 +素 +索 +紧 +紫 +累 +細 +終 +組 +結 +絡 +給 +絮 +統 +絲 +絶 +綁 +綉 +經 +綦 +綫 +維 +綱 +網 +緑 +線 +緣 +緯 +緻 +縣 +縫 +縮 +繁 +織 +繡 +繪 +繹 +纂 +纛 +纠 +红 +纣 +纤 +约 +级 +纨 +纪 +纫 +纬 +纭 +纯 +纰 +纱 +纲 +纳 +纵 +纶 +纷 +纸 +纹 +纺 +纽 +纾 +线 +绀 +绁 +绂 +练 +组 +绅 +细 +织 +终 +绉 +绊 +绋 +绌 +绍 +绎 +经 +绐 +绑 +绒 +结 +绔 +绕 +绗 +绘 +给 +绚 +绛 +络 +绝 +绞 +统 +绠 +绡 +绢 +绣 +绥 +绦 +继 +绨 +绩 +绪 +绫 +续 +绮 +绯 +绰 +绱 +绲 +绳 +维 +绵 +绶 +绷 +绸 +绺 +绻 +综 +绽 +绾 +绿 +缀 +缁 +缂 +缃 +缄 +缅 +缆 +缇 +缈 +缉 +缎 +缐 +缓 +缔 +缕 +编 +缘 +缙 +缚 +缜 +缝 +缠 +缢 +缤 +缥 +缦 +缨 +缩 +缪 +缫 +缬 +缭 +缮 +缰 +缴 +缸 +缺 +罂 +罄 +罐 +网 +罔 +罕 +罗 +罘 +罚 +罡 +罢 +罩 +罪 +置 +罱 +署 +罹 +羁 +羅 +羊 +羌 +美 +羔 +羙 +羚 +羞 +羟 +羡 +群 +羧 +義 +羯 +羰 +羲 +羸 +羹 +羽 +羿 +翁 +翅 +翊 +翌 +翎 +翏 +習 +翔 +翘 +翟 +翠 +翡 +翥 +翦 +翩 +翰 +翱 +翳 +翻 +翼 +耀 +老 +考 +耄 +者 +耆 +而 +耍 +耐 +耑 +耒 +耕 +耗 +耘 +耙 +耜 +耦 +耧 +耱 +耳 +耵 +耶 +耷 +耸 +耻 +耽 +耿 +聂 +聆 +聊 +聋 +职 +联 +聖 +聘 +聚 +聞 +聨 +聪 +聯 +聲 +職 +聿 +肃 +肄 +肆 +肇 +肉 +肋 +肌 +肓 +肖 +肘 +肚 +肛 +肝 +肟 +肠 +股 +肢 +肤 +肥 +肩 +肪 +肮 +肯 +肱 +育 +肴 +肺 +肼 +肽 +肾 +肿 +胀 +胁 +胃 +胄 +胆 +背 +胍 +胎 +胖 +胗 +胚 +胛 +胜 +胞 +胡 +胤 +胥 +胧 +胪 +胫 +胬 +胭 +胯 +胰 +胱 +胳 +胴 +胶 +胸 +胺 +能 +脂 +脆 +脉 +脊 +脍 +脏 +脐 +脑 +脒 +脓 +脖 +脘 +脚 +脯 +脱 +脲 +脸 +脾 +腆 +腈 +腊 +腋 +腌 +腐 +腑 +腓 +腔 +腕 +腥 +腦 +腩 +腭 +腮 +腰 +腱 +腴 +腸 +腹 +腺 +腻 +腼 +腾 +腿 +膀 +膈 +膊 +膏 +膘 +膚 +膛 +膜 +膝 +膠 +膦 +膨 +膳 +膺 +膻 +臀 +臂 +臃 +臆 +臊 +臣 +臧 +臨 +自 +臭 +至 +致 +臺 +臻 +臼 +臾 +舀 +舅 +舆 +與 +興 +舊 +舌 +舍 +舒 +舔 +舖 +舛 +舜 +舞 +舟 +航 +舫 +般 +舰 +舱 +舵 +舶 +舷 +舸 +船 +艄 +艇 +艘 +艦 +艮 +良 +艰 +色 +艳 +艶 +艷 +艹 +艺 +艾 +艿 +节 +芃 +芊 +芋 +芍 +芎 +芒 +芗 +芘 +芙 +芜 +芝 +芡 +芥 +芦 +芩 +芪 +芫 +芬 +芭 +芮 +芯 +花 +芳 +芷 +芸 +芹 +芽 +芾 +苁 +苄 +苇 +苋 +苍 +苎 +苏 +苑 +苒 +苓 +苔 +苕 +苗 +苘 +苛 +苜 +苝 +苞 +苟 +苡 +苣 +若 +苦 +苪 +苫 +苯 +英 +苴 +苷 +苹 +茁 +茂 +范 +茄 +茅 +茆 +茉 +茌 +茎 +茏 +茗 +茜 +茧 +茨 +茫 +茬 +茭 +茯 +茱 +茴 +茵 +茶 +茸 +茹 +茼 +荀 +荃 +荆 +草 +荏 +荐 +荒 +荔 +荘 +荚 +荞 +荟 +荠 +荡 +荣 +荤 +荥 +荧 +荨 +荪 +荫 +荬 +荭 +药 +荷 +荸 +荻 +荼 +荽 +莅 +莆 +莉 +莊 +莎 +莒 +莓 +莘 +莜 +莞 +莠 +莨 +莪 +莫 +莱 +莲 +莳 +莴 +获 +莹 +莺 +莼 +莽 +菀 +菁 +菅 +菇 +菊 +菌 +菏 +菓 +菖 +菘 +菜 +菟 +菠 +菡 +菩 +菪 +華 +菱 +菲 +菽 +萁 +萃 +萄 +萊 +萋 +萌 +萍 +萎 +萘 +萜 +萝 +萤 +营 +萦 +萧 +萨 +萬 +萭 +萱 +萸 +萼 +落 +葆 +葉 +葑 +葒 +著 +葛 +葡 +董 +葩 +葫 +葬 +葭 +葱 +葳 +葵 +葺 +蒂 +蒄 +蒋 +蒌 +蒙 +蒜 +蒟 +蒡 +蒤 +蒲 +蒸 +蒻 +蒽 +蒿 +蓄 +蓉 +蓋 +蓑 +蓓 +蓖 +蓝 +蓟 +蓥 +蓦 +蓬 +蓮 +蓼 +蓿 +蔑 +蔓 +蔗 +蔚 +蔟 +蔡 +蔫 +蔬 +蔷 +蔸 +蔺 +蔻 +蔼 +蔽 +蕃 +蕈 +蕉 +蕊 +蕒 +蕙 +蕤 +蕨 +蕲 +蕴 +蕹 +蕾 +薄 +薇 +薈 +薏 +薛 +薡 +薦 +薩 +薪 +薬 +薯 +薰 +薹 +藁 +藉 +藍 +藏 +藐 +藓 +藔 +藕 +藜 +藝 +藤 +藥 +藩 +藻 +藿 +蘆 +蘇 +蘑 +蘖 +蘭 +蘸 +蘼 +虎 +虏 +虐 +虑 +虔 +處 +虚 +虞 +號 +虢 +虫 +虬 +虱 +虹 +虻 +虽 +虾 +蚀 +蚁 +蚂 +蚊 +蚌 +蚓 +蚕 +蚜 +蚝 +蚣 +蚤 +蚧 +蚨 +蚩 +蚪 +蚬 +蚯 +蚱 +蚴 +蚶 +蛀 +蛆 +蛇 +蛊 +蛋 +蛎 +蛏 +蛐 +蛔 +蛙 +蛛 +蛟 +蛤 +蛭 +蛮 +蛰 +蛳 +蛴 +蛹 +蛾 +蜀 +蜂 +蜃 +蜇 +蜈 +蜊 +蜍 +蜒 +蜓 +蜕 +蜗 +蜘 +蜚 +蜛 +蜜 +蜡 +蜥 +蜱 +蜴 +蜷 +蜻 +蜿 +蝇 +蝈 +蝉 +蝌 +蝎 +蝗 +蝙 +蝠 +蝴 +蝶 +蝽 +蝾 +螂 +螃 +螈 +融 +螟 +螨 +螯 +螳 +螺 +蟀 +蟆 +蟋 +蟑 +蟒 +蟠 +蟹 +蟾 +蠄 +蠓 +蠔 +蠕 +蠖 +蠡 +蠢 +蠹 +血 +衅 +衆 +行 +衍 +術 +衔 +街 +衙 +衛 +衡 +衢 +衣 +补 +表 +衩 +衫 +衬 +衮 +衰 +衲 +衷 +衾 +衿 +袁 +袂 +袄 +袅 +袆 +袈 +袋 +袍 +袒 +袖 +袜 +袤 +袪 +被 +袭 +袱 +裁 +裂 +装 +裆 +裔 +裕 +裘 +裙 +補 +裝 +裟 +裡 +裤 +裨 +裱 +裳 +裴 +裸 +裹 +製 +褀 +褂 +複 +褐 +褒 +褓 +褔 +褚 +褥 +褪 +褰 +褲 +褶 +襁 +襄 +襟 +西 +要 +覃 +覆 +見 +覌 +視 +親 +覺 +觀 +见 +观 +规 +觅 +视 +觇 +览 +觉 +觊 +觎 +觐 +觑 +角 +觞 +解 +触 +觸 +言 +訂 +計 +訊 +訓 +記 +訣 +設 +許 +診 +訾 +詢 +試 +詩 +話 +該 +詹 +誉 +誊 +誌 +誓 +誘 +語 +誠 +説 +課 +調 +諾 +謇 +講 +謢 +謹 +證 +識 +譜 +警 +譬 +護 +讀 +變 +计 +订 +讣 +认 +讥 +讨 +让 +讪 +讫 +训 +议 +讯 +记 +讲 +讳 +讴 +讶 +讷 +许 +讹 +论 +讼 +讽 +设 +访 +诀 +证 +诃 +评 +诅 +识 +诈 +诉 +诊 +诋 +词 +诏 +译 +诒 +试 +诗 +诘 +诙 +诚 +诛 +话 +诞 +诟 +诠 +诡 +询 +诣 +诤 +该 +详 +诧 +诩 +诫 +诬 +语 +误 +诰 +诱 +诲 +说 +诵 +请 +诸 +诺 +读 +诽 +课 +诿 +谀 +谁 +调 +谄 +谅 +谆 +谈 +谊 +谋 +谌 +谍 +谎 +谏 +谐 +谑 +谒 +谓 +谔 +谕 +谗 +谙 +谚 +谛 +谜 +谟 +谡 +谢 +谣 +谤 +谥 +谦 +谧 +谨 +谩 +谪 +谬 +谭 +谯 +谰 +谱 +谴 +谷 +豁 +豆 +豇 +豉 +豊 +豌 +豐 +豕 +豚 +象 +豪 +豫 +豹 +豺 +貂 +貅 +貉 +貊 +貌 +貔 +貝 +負 +財 +貢 +貨 +販 +貭 +貴 +買 +費 +貼 +貿 +賀 +賓 +賠 +賣 +質 +賴 +購 +贈 +贏 +贝 +贞 +负 +贡 +财 +责 +贤 +败 +账 +货 +质 +贩 +贪 +贫 +贬 +购 +贮 +贯 +贰 +贱 +贲 +贴 +贵 +贷 +贸 +费 +贺 +贻 +贼 +贾 +贿 +赁 +赂 +赃 +资 +赅 +赈 +赉 +赊 +赋 +赌 +赎 +赏 +赐 +赓 +赔 +赖 +赘 +赚 +赛 +赝 +赞 +赠 +赡 +赢 +赣 +赤 +赦 +赫 +赭 +走 +赳 +赴 +赵 +赶 +起 +趁 +超 +越 +趋 +趙 +趟 +趣 +足 +趴 +趵 +趸 +趾 +跃 +跄 +跆 +跋 +跌 +跎 +跑 +跖 +跚 +跛 +距 +跟 +跤 +跨 +跪 +路 +跳 +践 +跷 +跺 +跻 +踉 +踊 +踌 +踏 +踝 +踞 +踢 +踩 +踪 +踮 +踱 +踵 +踹 +蹂 +蹄 +蹇 +蹈 +蹉 +蹊 +蹋 +蹑 +蹒 +蹙 +蹦 +蹬 +蹭 +蹲 +蹴 +蹶 +蹼 +蹿 +躁 +躇 +躏 +身 +躬 +躯 +躲 +躺 +車 +軍 +軎 +軒 +軽 +載 +輕 +輝 +輪 +輯 +輸 +轉 +车 +轧 +轨 +轩 +转 +轭 +轮 +软 +轰 +轱 +轲 +轴 +轶 +轻 +轼 +载 +轿 +较 +辄 +辅 +辆 +辈 +辉 +辊 +辋 +辍 +辐 +辑 +输 +辕 +辖 +辗 +辘 +辙 +辛 +辜 +辞 +辟 +辣 +辦 +辧 +辨 +辩 +辫 +辰 +辱 +農 +边 +辽 +达 +迁 +迂 +迄 +迅 +过 +迈 +迎 +运 +近 +返 +还 +这 +进 +远 +违 +连 +迟 +迢 +迤 +迥 +迦 +迩 +迪 +迫 +迭 +迮 +述 +迳 +迷 +迸 +迹 +追 +退 +送 +适 +逃 +逄 +逅 +逆 +选 +逊 +逍 +透 +逐 +逑 +递 +途 +逗 +這 +通 +逛 +逝 +逞 +速 +造 +逢 +連 +逭 +逮 +逯 +進 +逵 +逸 +逹 +逺 +逻 +逼 +逾 +遁 +遂 +遇 +運 +遍 +過 +遏 +遐 +遒 +道 +達 +遗 +遛 +遠 +遢 +遣 +遥 +遨 +適 +遭 +遮 +遴 +遵 +選 +避 +邀 +邂 +邃 +還 +邈 +邋 +邏 +邑 +邓 +邕 +邗 +邛 +邝 +邡 +邢 +那 +邦 +邨 +邪 +邬 +邮 +邯 +邰 +邱 +邳 +邴 +邵 +邸 +邹 +邺 +邻 +郁 +郅 +郊 +郎 +郏 +郑 +郓 +郗 +郛 +郜 +郝 +郡 +郢 +郦 +郧 +部 +郫 +郭 +郯 +郴 +郵 +郸 +都 +郾 +鄂 +鄄 +鄉 +鄙 +鄞 +鄢 +鄣 +鄭 +鄯 +鄱 +鄺 +酉 +酊 +酋 +酌 +配 +酐 +酒 +酗 +酚 +酝 +酞 +酡 +酣 +酥 +酪 +酬 +酮 +酯 +酰 +酱 +酵 +酶 +酷 +酸 +酿 +醇 +醉 +醋 +醌 +醍 +醐 +醒 +醚 +醛 +醜 +醪 +醫 +醬 +醴 +醺 +釀 +采 +釉 +释 +里 +重 +野 +量 +金 +釜 +針 +釣 +鈴 +鈺 +鉅 +鉌 +鉴 +銀 +銘 +銭 +銮 +銷 +鋪 +鋼 +錄 +錦 +錯 +錶 +鍊 +鍋 +鍵 +鎏 +鎖 +鎮 +鎻 +鏖 +鏡 +鏢 +鐘 +鐡 +鐵 +鑒 +鑫 +鑽 +钇 +针 +钉 +钊 +钎 +钏 +钒 +钓 +钕 +钗 +钙 +钚 +钛 +钜 +钝 +钞 +钟 +钠 +钡 +钢 +钣 +钥 +钦 +钧 +钨 +钩 +钬 +钮 +钯 +钰 +钱 +钳 +钴 +钵 +钺 +钻 +钼 +钽 +钾 +钿 +铀 +铁 +铂 +铃 +铄 +铅 +铆 +铈 +铉 +铊 +铋 +铌 +铍 +铎 +铐 +铑 +铒 +铕 +铖 +铛 +铜 +铝 +铞 +铟 +铠 +铡 +铢 +铣 +铤 +铧 +铨 +铪 +铬 +铭 +铮 +铰 +铱 +铲 +铵 +银 +铸 +铺 +链 +铿 +销 +锁 +锂 +锄 +锅 +锆 +锈 +锉 +锋 +锌 +锏 +锐 +锑 +锔 +锕 +锗 +锘 +错 +锚 +锛 +锝 +锟 +锡 +锢 +锣 +锤 +锥 +锦 +锨 +锭 +键 +锯 +锰 +锲 +锴 +锵 +锶 +锷 +锹 +锺 +锻 +锽 +镀 +镁 +镂 +镇 +镉 +镊 +镌 +镍 +镐 +镑 +镒 +镓 +镖 +镗 +镛 +镜 +镝 +镣 +镦 +镧 +镭 +镯 +镰 +镱 +镳 +镶 +長 +长 +門 +閃 +開 +閏 +閑 +間 +閟 +関 +閣 +閥 +閩 +關 +门 +闪 +闫 +闭 +问 +闯 +闰 +闱 +闲 +闳 +间 +闵 +闷 +闸 +闹 +闺 +闻 +闽 +闾 +闿 +阀 +阁 +阂 +阄 +阅 +阆 +阈 +阉 +阊 +阎 +阐 +阑 +阔 +阕 +阖 +阗 +阙 +阚 +阜 +阝 +队 +阡 +阪 +阮 +阱 +防 +阳 +阴 +阵 +阶 +阻 +阿 +陀 +陂 +附 +际 +陆 +陇 +陈 +陉 +陋 +陌 +降 +限 +陕 +陛 +陟 +陡 +院 +除 +陨 +险 +陪 +陰 +陲 +陳 +陵 +陶 +陷 +陽 +隅 +隆 +隋 +隍 +随 +隐 +隔 +隗 +隘 +隙 +際 +障 +隧 +隨 +隶 +隹 +隼 +隽 +难 +雀 +雁 +雄 +雅 +集 +雇 +雉 +雌 +雍 +雎 +雏 +雒 +雕 +雙 +雜 +雞 +離 +難 +雨 +雪 +雯 +雲 +雳 +零 +雷 +雹 +電 +雾 +需 +霁 +霄 +霆 +震 +霉 +霍 +霎 +霏 +霓 +霖 +霜 +霞 +霭 +霰 +露 +霸 +霹 +霾 +靈 +靑 +青 +靓 +靖 +静 +靛 +靜 +非 +靠 +靡 +面 +靥 +革 +靳 +靴 +靶 +鞅 +鞋 +鞍 +鞑 +鞘 +鞠 +鞣 +鞭 +韓 +韦 +韧 +韩 +韫 +韬 +韭 +音 +韵 +韶 +頂 +順 +頓 +頔 +頡 +頤 +頭 +頰 +題 +顏 +顔 +類 +顧 +顯 +页 +顶 +顷 +项 +顺 +须 +顽 +顾 +顿 +颁 +颂 +预 +颅 +领 +颇 +颈 +颉 +颊 +颌 +颍 +颐 +频 +颓 +颔 +颖 +颗 +题 +颚 +颛 +颜 +额 +颞 +颠 +颡 +颢 +颤 +颦 +颧 +風 +飄 +风 +飒 +飓 +飕 +飘 +飙 +飛 +飞 +食 +飨 +飬 +飮 +飯 +飲 +飼 +飾 +餅 +養 +餐 +餘 +餠 +館 +餮 +饃 +饅 +饕 +饥 +饨 +饪 +饬 +饭 +饮 +饯 +饰 +饱 +饲 +饴 +饵 +饶 +饷 +饸 +饹 +饺 +饼 +饽 +饿 +馁 +馄 +馅 +馆 +馈 +馊 +馋 +馍 +馏 +馒 +馕 +首 +馗 +香 +馥 +馨 +馬 +馮 +馴 +駕 +騰 +騷 +驗 +马 +驭 +驮 +驯 +驰 +驱 +驳 +驴 +驶 +驷 +驸 +驹 +驻 +驼 +驽 +驾 +驿 +骁 +骂 +骄 +骅 +骆 +骇 +骊 +骋 +验 +骏 +骐 +骑 +骓 +骗 +骚 +骛 +骜 +骝 +骞 +骠 +骡 +骤 +骥 +骧 +骨 +骰 +骶 +骷 +骸 +骺 +骼 +髁 +髂 +髅 +髋 +髌 +髓 +體 +高 +髙 +髦 +髪 +髮 +髯 +髻 +鬃 +鬓 +鬟 +鬣 +鬲 +鬼 +魁 +魂 +魄 +魅 +魇 +魉 +魏 +魔 +魚 +鮮 +鯉 +鯨 +鯽 +鱬 +鱸 +鱻 +鱼 +鱿 +鲁 +鲂 +鲅 +鲆 +鲇 +鲈 +鲍 +鲑 +鲛 +鲜 +鲟 +鲢 +鲤 +鲨 +鲩 +鲫 +鲮 +鲱 +鲲 +鲳 +鲵 +鲷 +鲸 +鲽 +鳃 +鳄 +鳅 +鳌 +鳍 +鳎 +鳐 +鳕 +鳖 +鳗 +鳙 +鳜 +鳝 +鳞 +鳟 +鳥 +鳯 +鳳 +鴨 +鴻 +鴿 +鵬 +鶏 +鷄 +鷗 +鷹 +鸟 +鸠 +鸡 +鸢 +鸣 +鸥 +鸦 +鸨 +鸪 +鸬 +鸭 +鸯 +鸳 +鸵 +鸽 +鸾 +鸿 +鹂 +鹃 +鹄 +鹅 +鹉 +鹊 +鹌 +鹏 +鹑 +鹗 +鹜 +鹞 +鹤 +鹦 +鹧 +鹫 +鹭 +鹰 +鹳 +鹽 +鹿 +麂 +麋 +麒 +麓 +麗 +麝 +麟 +麥 +麦 +麯 +麵 +麸 +麺 +麻 +麽 +麾 +黃 +黄 +黍 +黎 +黏 +黑 +黒 +黔 +默 +黛 +黜 +黝 +點 +黟 +黠 +黯 +鼎 +鼓 +鼙 +鼠 +鼬 +鼻 +鼽 +鼾 +齊 +齋 +齐 +齿 +龄 +龈 +龊 +龋 +龌 +龍 +龙 +龚 +龛 +龟 +龢 +︽ +︾ +﹏ +﹥ +﹪ +! +# +% +& +( +) ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; += +? +@ +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +| +~ +¥ diff --git a/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/rec.onnx b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/rec.onnx new file mode 100644 index 00000000..8b1a391b Binary files /dev/null and b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/rec.onnx differ diff --git a/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/sample.jpg b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/sample.jpg new file mode 100644 index 00000000..35de3514 Binary files /dev/null and b/deploy/ocr/vendor/lw.PPOCR.C/models/ppocrv6-tiny/sample.jpg differ diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/nhwc_depthwise.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/nhwc_depthwise.c new file mode 100644 index 00000000..61d7bf78 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/nhwc_depthwise.c @@ -0,0 +1,41 @@ +#include "nhwc_internal.h" + +#include +#include + +int lw_nhwc_depthwise_packed_weight_count(uint32_t channels, + uint32_t kernel_h, + uint32_t kernel_w, + uint64_t* element_count) { + uint64_t blocks; + uint64_t taps; + if (channels == 0u || kernel_h == 0u || kernel_w == 0u || element_count == NULL) return 0; + blocks = ((uint64_t)channels + LW_NHWC_DEPTHWISE_BLOCK - 1u) / LW_NHWC_DEPTHWISE_BLOCK; + taps = (uint64_t)kernel_h * kernel_w; + if (blocks > UINT64_MAX / taps || blocks * taps > UINT64_MAX / LW_NHWC_DEPTHWISE_BLOCK) return 0; + *element_count = blocks * taps * LW_NHWC_DEPTHWISE_BLOCK; + return *element_count <= SIZE_MAX / sizeof(float); +} + +void lw_pack_nhwc_depthwise_f32(const float* weights, uint32_t channels, + uint32_t kernel_h, uint32_t kernel_w, + float* packed_weights) { + uint32_t blocks; + uint32_t taps; + if (weights == NULL || packed_weights == NULL || channels == 0u || kernel_h == 0u || kernel_w == 0u) return; + blocks = (channels + LW_NHWC_DEPTHWISE_BLOCK - 1u) / LW_NHWC_DEPTHWISE_BLOCK; + taps = kernel_h * kernel_w; + for (uint32_t block = 0u; block < blocks; ++block) { + for (uint32_t tap = 0u; tap < taps; ++tap) { + uint32_t ky = tap / kernel_w; + uint32_t kx = tap - ky * kernel_w; + for (uint32_t lane = 0u; lane < LW_NHWC_DEPTHWISE_BLOCK; ++lane) { + uint32_t channel = block * LW_NHWC_DEPTHWISE_BLOCK + lane; + size_t dst = ((size_t)block * taps + tap) * LW_NHWC_DEPTHWISE_BLOCK + lane; + packed_weights[dst] = channel < channels + ? weights[((size_t)channel * kernel_h + ky) * kernel_w + kx] + : 0.0f; + } + } + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/nhwc_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/nhwc_internal.h new file mode 100644 index 00000000..47e1d936 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/nhwc_internal.h @@ -0,0 +1,251 @@ +#ifndef LW_NHWC_INTERNAL_H +#define LW_NHWC_INTERNAL_H + +#include "lw_infer.h" + +#include + +#define LW_NHWC_OC_BLOCK 16u +#define LW_NHWC_DEPTHWISE_BLOCK 32u +#define LW_NHWC_PIXEL_TILE 6u +#define LW_NHWC_POINTWISE_GROUP_TILES 8u +#define LW_NHWC_DENSE_KC 512u +#define LW_NHWC_PARTIAL_FLOATS (LW_NHWC_PIXEL_TILE * LW_NHWC_OC_BLOCK) + +typedef enum lw_nhwc_activation { + LW_NHWC_ACT_NONE = 0, + LW_NHWC_ACT_RELU = 1, + LW_NHWC_ACT_HARDSWISH = 2, + LW_NHWC_ACT_GELU = 3 +} lw_nhwc_activation; + +typedef struct lw_nhwc_epilogue { + const float* bias; + const float* residual; + uint16_t activation; + uint16_t reserved; + float alpha; + float beta; + /* Optional channel bias folded from a separate Add node. Applied to the + * finished accumulator AFTER accumulation (and before residual and + * activation), preserving the bit-exact conv -> add -> ... ordering. */ + const float* post_bias; +} lw_nhwc_epilogue; + +float lw_nhwc_gelu_scalar_exact_f32(float value); +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) +#include +__m256 lw_nhwc_avx2_gelu_vector_exact_f32(__m256 value); +#endif + +int lw_nhwc_dense_packed_weight_count(uint32_t input_channels, + uint32_t output_channels, + uint32_t kernel_h, + uint32_t kernel_w, + uint64_t* element_count); + +void lw_pack_nhwc_dense_f32(const float* weights, + uint32_t input_channels, + uint32_t output_channels, + uint32_t kernel_h, + uint32_t kernel_w, + float* packed_weights); + +typedef struct lw_nhwc_dense_desc { + uint32_t batch; + uint32_t input_channels; + uint32_t input_height; + uint32_t input_width; + uint32_t output_channels; + uint32_t output_height; + uint32_t output_width; + uint32_t kernel_h; + uint32_t kernel_w; + uint32_t stride_h; + uint32_t stride_w; + uint32_t pad_top; + uint32_t pad_left; + uint32_t pad_bottom; + uint32_t pad_right; + uint32_t dense_kc; + /* Optional sharded-row range: output rows [offset, offset + output_height) + * of the full output. Zero keeps the whole-output serial semantics. */ + uint32_t output_row_offset; +} lw_nhwc_dense_desc; + +int lw_nhwc_dense_scratch_bytes(const lw_nhwc_dense_desc* desc, uint64_t* scratch_bytes); + +int lw_nhwc_dense_prepared_scratch_bytes(const lw_nhwc_dense_desc* desc, uint64_t* scratch_bytes); + +lw_status lw_avx2_fma_nhwc_dense_prepared_f32(const float* input, + const float* packed_weights, + const int32_t* input_offsets_k, + const int32_t* patch_offsets_k, + const lw_nhwc_epilogue* epilogue, + float* output, + const lw_nhwc_dense_desc* desc, + void* scratch, + uint64_t scratch_bytes); + +lw_status lw_avx2_fma_nhwc_dense_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + const lw_nhwc_dense_desc* desc, + void* scratch, + uint64_t scratch_bytes); +/* Input/output are NHWC. This API is experimental and production-disabled. */ +typedef struct lw_nhwc_depthwise_desc { + uint32_t batch; + uint32_t channels; + uint32_t input_height; + uint32_t input_width; + uint32_t output_height; + uint32_t output_width; + uint32_t kernel_h; + uint32_t kernel_w; + uint32_t stride_h; + uint32_t stride_w; + uint32_t pad_top; + uint32_t pad_left; + uint32_t pad_bottom; + uint32_t pad_right; + /* Optional sharded-row range: output rows [offset, offset + output_height) + * of the full output. Zero keeps the whole-output serial semantics. */ + uint32_t output_row_offset; + /* Optional channel bias folded from a separate Add node: the accumulator + * starts at zero (bias stays NULL) and post_bias is added lane-exactly + * before the store, matching a standalone Add pass bit for bit. */ + const float* post_bias; +} lw_nhwc_depthwise_desc; + +int lw_nhwc_depthwise_packed_weight_count(uint32_t channels, + uint32_t kernel_h, + uint32_t kernel_w, + uint64_t* element_count); + +void lw_pack_nhwc_depthwise_f32(const float* weights, + uint32_t channels, + uint32_t kernel_h, + uint32_t kernel_w, + float* packed_weights); + +lw_status lw_avx2_fma_nhwc_depthwise_f32(const float* input, + const float* packed_weights, + const float* bias, + float* output, + const lw_nhwc_depthwise_desc* desc); +typedef struct lw_nhwc_depthwise_stats { + uint64_t x1_invocations; + uint64_t x2_invocations; +} lw_nhwc_depthwise_stats; + +lw_status lw_avx2_fma_nhwc_depthwise_profiled_f32(const float* input, + const float* packed_weights, + const float* bias, + float* output, + const lw_nhwc_depthwise_desc* desc, + lw_nhwc_depthwise_stats* stats); +void lw_avx2_fma_nhwc_pointwise_grouped_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels, + uint32_t group_tiles); + +void lw_avx2_fma_nhwc_pointwise_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels); +void lw_avx2_fma_nhwc_pointwise_4x16_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels); +void lw_avx2_fma_nhwc_pointwise_3x32_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels); +void lw_avx2_fma_nhwc_pointwise_2x32_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels); + +void lw_avx2_nhwc_affine_f32(const float* input, const float* mul, const float* add, + float* output, uint32_t pixels, uint32_t channels); +void lw_avx2_nchw_affine_f32(const float* input, const float* mul, const float* add, + float* output, uint32_t channels, uint32_t spatial); +void lw_avx2_nhwc_reduce_mean_hw_f32(const float* input, float* output, + uint32_t batch, uint32_t height, + uint32_t width, uint32_t channels); +/* Channel-group variant for sharded execution: processes channels + * [0, channels) of a tensor whose rows are input_stride wide, writing to + * output rows of output_stride. Per-channel pixel visit order is identical + * to lw_avx2_nhwc_reduce_mean_hw_f32, so results are bit-identical. */ +void lw_avx2_nhwc_reduce_mean_hw_strided_f32(const float* input, float* output, + uint32_t batch, uint32_t height, + uint32_t width, uint32_t channels, + uint32_t input_stride, uint32_t output_stride); +void lw_avx2_nhwc_pool_f32(const float* input, float* output, + uint32_t batch, uint32_t input_height, + uint32_t input_width, uint32_t output_height, + uint32_t output_width, uint32_t channels, + uint32_t kernel_h, uint32_t kernel_w, + uint32_t stride_h, uint32_t stride_w, + uint32_t pad_top, uint32_t pad_left, + uint8_t count_include_pad, uint8_t is_max); +/* Integer-scale nearest resize (planner-guaranteed scales): each output row + * is a whole-row copy of the source row at oy / scale_h, each source pixel + * repeated scale_w times. Pure data movement, bit-identical to the scalar + * nearest path. */ +void lw_avx2_nhwc_resize_nearest_f32(const float* input, float* output, + uint32_t batch, uint32_t channels, + uint32_t input_height, uint32_t input_width, + uint32_t output_height, uint32_t output_width); +/* 2x2 stride-2 no-pad ConvTranspose as four per-tap 1x1 GEMMs. Weights are + * packed [tap][oc/16][ic][16] from the ONNX [ic, oc, 2, 2] layout. The + * epilogue supports bias and ReLU only; Sigmoid is deliberately excluded so + * callers can defuse it into a separate bit-identical pass. */ +typedef struct lw_nhwc_convtranspose_desc { + uint32_t batch; + uint32_t input_channels; + uint32_t input_height; + uint32_t input_width; + uint32_t output_channels; + uint32_t output_height; + uint32_t output_width; +} lw_nhwc_convtranspose_desc; + +int lw_nhwc_convtranspose_packed_weight_count(uint32_t input_channels, + uint32_t output_channels, + uint64_t* element_count); +void lw_pack_nhwc_convtranspose2x2_f32(const float* weights, uint32_t input_channels, + uint32_t output_channels, float* packed_weights); +lw_status lw_wasm128_nhwc_convtranspose2x2_s2_f32( + const float* input, const float* packed_weights, const lw_nhwc_epilogue* epilogue, + float* output, const lw_nhwc_convtranspose_desc* desc); +lw_status lw_wasm128_nhwc_convtranspose2x2_s2_c1_f32( + const float* input, const float* weights, const lw_nhwc_epilogue* epilogue, + float* output, const lw_nhwc_convtranspose_desc* desc); +lw_status lw_avx2_fma_nhwc_convtranspose2x2_s2_f32( + const float* input, const float* packed_weights, const lw_nhwc_epilogue* epilogue, + float* output, const lw_nhwc_convtranspose_desc* desc); +/* Single-output-channel probability-map variant; unpaced ONNX weights + * [ic, 1, 2, 2], ReLU-only epilogue. */ +lw_status lw_avx2_fma_nhwc_convtranspose2x2_s2_c1_f32( + const float* input, const float* weights, const lw_nhwc_epilogue* epilogue, + float* output, const lw_nhwc_convtranspose_desc* desc); +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/nhwc_pack.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/nhwc_pack.c new file mode 100644 index 00000000..44e06bf6 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/nhwc_pack.c @@ -0,0 +1,79 @@ +#include "nhwc_internal.h" + +#include +#include + +int lw_nhwc_dense_packed_weight_count(uint32_t input_channels, + uint32_t output_channels, + uint32_t kernel_h, + uint32_t kernel_w, + uint64_t* element_count) { + uint64_t output_blocks; + uint64_t count; + + if (input_channels == 0u || output_channels == 0u || kernel_h == 0u || + kernel_w == 0u || element_count == NULL) { + return 0; + } + output_blocks = ((uint64_t)output_channels + LW_NHWC_OC_BLOCK - 1u) / + LW_NHWC_OC_BLOCK; + count = output_blocks; + if (count > UINT64_MAX / kernel_h) { + return 0; + } + count *= kernel_h; + if (count > UINT64_MAX / kernel_w) { + return 0; + } + count *= kernel_w; + if (count > UINT64_MAX / input_channels) { + return 0; + } + count *= input_channels; + if (count > UINT64_MAX / LW_NHWC_OC_BLOCK) { + return 0; + } + count *= LW_NHWC_OC_BLOCK; + if (count > (uint64_t)(SIZE_MAX / sizeof(float))) { + return 0; + } + *element_count = count; + return 1; +} + +void lw_pack_nhwc_dense_f32(const float* weights, + uint32_t input_channels, + uint32_t output_channels, + uint32_t kernel_h, + uint32_t kernel_w, + float* packed_weights) { + const uint32_t output_blocks = + output_channels / LW_NHWC_OC_BLOCK + + (output_channels % LW_NHWC_OC_BLOCK == 0u ? 0u : 1u); + const uint32_t taps = kernel_h * kernel_w; + uint32_t output_block; + + for (output_block = 0u; output_block < output_blocks; ++output_block) { + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + uint32_t tap; + for (tap = 0u; tap < taps; ++tap) { + const uint32_t kh = tap / kernel_w; + const uint32_t kw = tap - kh * kernel_w; + uint32_t lane; + for (lane = 0u; lane < LW_NHWC_OC_BLOCK; ++lane) { + const uint64_t output_channel = + (uint64_t)output_block * LW_NHWC_OC_BLOCK + lane; + const uint64_t packed_index = + ((((uint64_t)output_block * input_channels + input_channel) * taps + + tap) * LW_NHWC_OC_BLOCK) + lane; + packed_weights[(size_t)packed_index] = + output_channel < output_channels + ? weights[((((output_channel * input_channels + input_channel) * + kernel_h + kh) * kernel_w) + kw)] + : 0.0f; + } + } + } + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv.c new file mode 100644 index 00000000..fd54c94c --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv.c @@ -0,0 +1,242 @@ +#include "packed_conv_internal.h" + +/* NCHW pointwise-Conv packing and the portable microkernel fallback. */ + +#include "cpu_features.h" +#include "simd_kernels.h" + +#include +#include + +int lw_packed_conv1x1_weight_count(uint32_t input_channels, uint32_t output_channels, + uint64_t* weight_count) { + uint64_t output_blocks; + if (input_channels == 0u || output_channels == 0u || weight_count == NULL) { + return 0; + } + output_blocks = ((uint64_t)output_channels + LW_PACKED_CONV1X1_OUTPUT_TILE - 1u) / + LW_PACKED_CONV1X1_OUTPUT_TILE; + if (output_blocks > UINT64_MAX / input_channels || + output_blocks * input_channels > UINT64_MAX / LW_PACKED_CONV1X1_OUTPUT_TILE) { + return 0; + } + *weight_count = output_blocks * input_channels * LW_PACKED_CONV1X1_OUTPUT_TILE; + return *weight_count <= (uint64_t)(SIZE_MAX / sizeof(float)); +} + +void lw_pack_conv1x1_weights_f32(const float* weights, uint32_t input_channels, + uint32_t output_channels, float* packed_weights) { + uint32_t output_block; + uint32_t output_blocks = + (output_channels + LW_PACKED_CONV1X1_OUTPUT_TILE - 1u) / LW_PACKED_CONV1X1_OUTPUT_TILE; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + uint32_t output_lane; + for (output_lane = 0u; output_lane < LW_PACKED_CONV1X1_OUTPUT_TILE; ++output_lane) { + uint32_t output_channel = + output_block * LW_PACKED_CONV1X1_OUTPUT_TILE + output_lane; + uint64_t packed_index = ((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE + + output_lane; + packed_weights[(size_t)packed_index] = + output_channel < output_channels + ? weights[(size_t)((uint64_t)output_channel * input_channels + + input_channel)] + : 0.0f; + } + } + } +} + +void lw_scalar_packed_conv1x1_f32(const float* input, const float* packed_weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + uint32_t input_channels = (uint32_t)input_dimensions[1]; + uint32_t output_channels = (uint32_t)output_dimensions[1]; + uint64_t channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + uint32_t output_blocks = + (output_channels + LW_PACKED_CONV1X1_OUTPUT_TILE - 1u) / LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_block; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + uint32_t output_base = output_block * LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t valid_outputs = output_channels - output_base; + uint64_t spatial; + if (valid_outputs > LW_PACKED_CONV1X1_OUTPUT_TILE) { + valid_outputs = LW_PACKED_CONV1X1_OUTPUT_TILE; + } + for (spatial = 0u; spatial < channel_plane; ++spatial) { + float accumulators[LW_PACKED_CONV1X1_OUTPUT_TILE] = {0.0f, 0.0f, 0.0f, 0.0f}; + uint32_t output_lane; + uint32_t input_channel; + for (output_lane = 0u; output_lane < valid_outputs; ++output_lane) { + accumulators[output_lane] = + bias == NULL ? 0.0f : bias[output_base + output_lane]; + } + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + float input_value = + batch_input[(size_t)((uint64_t)input_channel * channel_plane + spatial)]; + const float* packed = + packed_weights + + (size_t)(((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE); + for (output_lane = 0u; output_lane < valid_outputs; ++output_lane) { + accumulators[output_lane] += input_value * packed[output_lane]; + } + } + for (output_lane = 0u; output_lane < valid_outputs; ++output_lane) { + batch_output[(size_t)(((uint64_t)output_base + output_lane) * channel_plane + + spatial)] = accumulators[output_lane]; + } + } + } + } +} + +/* Large Small/Medium REC pointwise layers dominate their graph time. The + * remaining FMA candidates stay behind the experimental dispatch switch. */ +static uint32_t lw_large_rec_fma_variant(const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + uint32_t input_channels; + uint32_t output_channels; + uint32_t height; + if (input_dimensions == NULL || output_dimensions == NULL || + input_dimensions[0] != 1 || output_dimensions[0] != 1 || + input_dimensions[2] <= 0 || input_dimensions[3] <= 0 || + output_dimensions[2] != input_dimensions[2] || + output_dimensions[3] != input_dimensions[3]) { + return 0u; + } + input_channels = (uint32_t)input_dimensions[1]; + output_channels = (uint32_t)output_dimensions[1]; + height = (uint32_t)input_dimensions[2]; + if (height == 6u && (uint32_t)input_dimensions[3] >= 80u) { + if ((input_channels == 192u && output_channels == 384u) || + (input_channels == 384u && output_channels == 192u)) { + return 4u; + } + if ((input_channels == 512u && output_channels == 1024u) || + (input_channels == 1024u && output_channels == 512u)) { + return 8u; + } + } + if (height == 3u && (uint32_t)input_dimensions[3] >= 160u && + ((input_channels == 768u && output_channels == 384u) || + (input_channels == 1536u && output_channels == 768u))) { + return 8u; + } + return 0u; +} + +#if defined(LW_EXPERIMENTAL_AVX2_FMA_CONV1X1_DISPATCH) +/* + * FMA can lower the CPU frequency on some x64 hosts. Keep the measured + * medium/late shapes on the regular AVX2 path until a wider calibration set + * proves otherwise. Only shapes with repeatable wins are enabled below; unknown + * shapes remain on regular AVX2. + */ +static int lw_experimental_fma_shape_allowed(const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + uint32_t input_channels; + uint32_t output_channels; + uint32_t height; + if (input_dimensions == NULL || output_dimensions == NULL || input_dimensions[2] <= 0) { + return 0; + } + input_channels = (uint32_t)input_dimensions[1]; + output_channels = (uint32_t)output_dimensions[1]; + height = (uint32_t)input_dimensions[2]; + if (height == 6u && input_channels == 1024u && output_channels == 512u) { + return 0; + } + if ((height == 12u && + ((input_channels == 96u && output_channels == 192u) || + (input_channels == 48u && output_channels == 96u) || + (input_channels == 96u && output_channels == 48u))) || + (height == 6u && + ((input_channels == 96u && output_channels == 192u) || + (input_channels == 192u && output_channels == 96u) || + (input_channels == 192u && output_channels == 384u) || + (input_channels == 512u && output_channels == 1024u))) || + (height == 3u && + ((input_channels == 160u && output_channels == 320u) || + (input_channels == 384u && output_channels == 768u) || + (input_channels == 768u && output_channels == 384u) || + (input_channels == 1536u && output_channels == 768u)))) { + return 1; + } + return 0; +} +#endif +#if defined(LW_EXPERIMENTAL_AVX2_FMA_CONV1X1_DISPATCH) +/* + * The 8x8 candidate keeps two packed four-output blocks live. It is useful + * only for the large late feature maps where the extra output reuse offsets + * the additional register pressure. Keep this list explicit: the regular + * four-output FMA kernel remains the fallback for every other shape. + */ +static int lw_experimental_fma_8x8_shape_allowed( + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { + uint32_t input_channels; + uint32_t output_channels; + uint32_t height; + if (input_dimensions == NULL || output_dimensions == NULL || input_dimensions[2] <= 0) { + return 0; + } + input_channels = (uint32_t)input_dimensions[1]; + output_channels = (uint32_t)output_dimensions[1]; + height = (uint32_t)input_dimensions[2]; + return (height == 6u && input_channels == 512u && output_channels == 1024u) || + (height == 3u && + ((input_channels == 768u && output_channels == 384u) || + (input_channels == 1536u && output_channels == 768u))); +} +#endif + +void lw_packed_conv1x1_f32(const float* input, const float* packed_weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + const lw_cpu_capabilities capabilities = lw_get_cpu_capabilities(); + const lw_simd_level simd_level = capabilities.simd; + uint32_t fma_variant = lw_large_rec_fma_variant(input_dimensions, output_dimensions); +#if defined(LW_EXPERIMENTAL_AVX2_FMA_CONV1X1_DISPATCH) + if (fma_variant == 0u && + lw_experimental_fma_shape_allowed(input_dimensions, output_dimensions)) { + fma_variant = lw_experimental_fma_8x8_shape_allowed(input_dimensions, + output_dimensions) ? 8u : 4u; + } +#endif + if (capabilities.has_avx2_fma && fma_variant != 0u) { + if (fma_variant == 8u) { + lw_avx2_fma_packed_conv1x1_8x8_f32(input, packed_weights, bias, output, + input_dimensions, output_dimensions); + } else { + lw_avx2_fma_packed_conv1x1_f32(input, packed_weights, bias, output, + input_dimensions, output_dimensions); + } + return; + } + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_packed_conv1x1_f32(input, packed_weights, bias, output, input_dimensions, + output_dimensions); + } else if (lw_simd_level_is_neon(simd_level)) { + lw_neon_packed_conv1x1_f32(input, packed_weights, bias, output, input_dimensions, + output_dimensions); + } else if (lw_simd_level_is_lsx(simd_level)) { + lw_lsx_packed_conv1x1_f32(input, packed_weights, bias, output, input_dimensions, + output_dimensions); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_packed_conv1x1_f32(input, packed_weights, bias, output, input_dimensions, + output_dimensions); + } else { + lw_scalar_packed_conv1x1_f32(input, packed_weights, bias, output, input_dimensions, + output_dimensions); + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv3x3.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv3x3.c new file mode 100644 index 00000000..be9891f9 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv3x3.c @@ -0,0 +1,178 @@ +#include "packed_conv3x3_internal.h" + +#include "cpu_features.h" +#include "simd_kernels.h" + +#include + +int lw_packed_conv3x3_stride2_weight_count(uint32_t input_channels, + uint32_t output_channels, + uint64_t* weight_count) { + uint64_t output_blocks; + if (input_channels == 0u || output_channels == 0u || weight_count == NULL) { + return 0; + } + output_blocks = + ((uint64_t)output_channels + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE - 1u) / + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + if (output_blocks > UINT64_MAX / input_channels || + output_blocks * input_channels > UINT64_MAX / 9u || + output_blocks * input_channels * 9u > + UINT64_MAX / LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE) { + return 0; + } + *weight_count = output_blocks * input_channels * 9u * + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + return *weight_count <= (uint64_t)(SIZE_MAX / sizeof(float)); +} + +void lw_pack_conv3x3_stride2_weights_f32(const float* weights, uint32_t input_channels, + uint32_t output_channels, float* packed_weights) { + const uint32_t output_blocks = + (output_channels + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE - 1u) / + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + uint32_t output_block; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + uint32_t kernel_index; + for (kernel_index = 0u; kernel_index < 9u; ++kernel_index) { + uint32_t output_lane; + for (output_lane = 0u; + output_lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + ++output_lane) { + const uint32_t output_channel = + output_block * LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE + output_lane; + const uint64_t packed_index = + (((uint64_t)output_block * input_channels + input_channel) * 9u + + kernel_index) * + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE + + output_lane; + packed_weights[(size_t)packed_index] = + output_channel < output_channels + ? weights[(size_t)(((uint64_t)output_channel * input_channels + + input_channel) * + 9u + + kernel_index)] + : 0.0f; + } + } + } + } +} + +void lw_scalar_packed_conv3x3_stride2_pad1_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t input_height = (uint32_t)input_dimensions[2]; + const uint32_t input_width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint32_t output_height = (uint32_t)output_dimensions[2]; + const uint32_t output_width = (uint32_t)output_dimensions[3]; + const uint64_t input_plane = (uint64_t)input_height * input_width; + const uint64_t output_plane = (uint64_t)output_height * output_width; + const uint32_t output_blocks = + (output_channels + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE - 1u) / + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * input_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * output_plane); + uint32_t output_block; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + const uint32_t output_base = + output_block * LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + uint32_t valid_outputs = output_channels - output_base; + uint32_t output_lane; + if (valid_outputs > LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE) { + valid_outputs = LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + } + for (output_lane = 0u; output_lane < valid_outputs; ++output_lane) { + uint32_t output_y; + for (output_y = 0u; output_y < output_height; ++output_y) { + uint32_t output_x; + for (output_x = 0u; output_x < output_width; ++output_x) { + float accumulator = + bias == NULL ? 0.0f : bias[output_base + output_lane]; + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * input_plane); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const int64_t input_y = + (int64_t)output_y * 2 - 1 + kernel_y; + uint32_t kernel_x; + if (input_y < 0 || input_y >= (int64_t)input_height) { + continue; + } + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const int64_t input_x = + (int64_t)output_x * 2 - 1 + kernel_x; + if (input_x >= 0 && input_x < (int64_t)input_width) { + const uint32_t kernel_index = kernel_y * 3u + kernel_x; + const uint64_t packed_index = + (((uint64_t)output_block * input_channels + + input_channel) * + 9u + + kernel_index) * + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE + + output_lane; + accumulator += + input_channel_data[(size_t)((uint64_t)(uint32_t)input_y * + input_width + + (uint32_t)input_x)] * + packed_weights[(size_t)packed_index]; + } + } + } + } + batch_output[(size_t)(((uint64_t)output_base + output_lane) * + output_plane + + (uint64_t)output_y * output_width + output_x)] = + accumulator; + } + } + } + } + } +} + +void lw_packed_conv3x3_stride2_pad1_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { + const lw_simd_level simd_level = lw_detect_simd_level(); +#if defined(LW_AVX2_FMA_CONV3X3_DISPATCH) + const lw_cpu_capabilities capabilities = lw_get_cpu_capabilities(); + const int tiny_rec_node6 = + input_dimensions[0] == 1 && input_dimensions[1] == 24 && + input_dimensions[2] == 24 && + (input_dimensions[3] == 160 || input_dimensions[3] == 480) && + output_dimensions[1] == 48 && output_dimensions[2] == 12 && + (output_dimensions[3] == 80 || output_dimensions[3] == 240); + const int small_rec_early = + input_dimensions[0] == 1 && input_dimensions[1] == 96 && + input_dimensions[2] == 24 && + (input_dimensions[3] == 320 || input_dimensions[3] == 480) && + output_dimensions[1] == 48 && output_dimensions[2] == 12 && + output_dimensions[3] == input_dimensions[3] / 2; + if (lw_simd_level_is_avx2(simd_level) && capabilities.has_avx2_fma && + (tiny_rec_node6 || small_rec_early)) { + lw_avx2_fma_packed_conv3x3_stride2_pad1_f32( + input, packed_weights, bias, output, input_dimensions, output_dimensions); + return; + } +#endif + if (lw_simd_level_is_avx2(simd_level) && + (uint32_t)output_dimensions[1] % LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE == 0u) { + lw_avx2_packed_conv3x3_stride2_pad1_f32( + input, packed_weights, bias, output, input_dimensions, output_dimensions); + } else { + lw_scalar_packed_conv3x3_stride2_pad1_f32( + input, packed_weights, bias, output, input_dimensions, output_dimensions); + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv3x3_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv3x3_internal.h new file mode 100644 index 00000000..641e847e --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv3x3_internal.h @@ -0,0 +1,30 @@ +#ifndef LW_PACKED_CONV3X3_INTERNAL_H +#define LW_PACKED_CONV3X3_INTERNAL_H + +/* + * Private packed-weight contract for NCHW group-1 3x3 Conv with stride 2 and + * pad 1. LWM keeps canonical OIHW weights; eligible sessions may convert them + * once to [OC/8][IC][3][3][8]. + */ + +#include + +#define LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE 8u + +typedef void (*lw_packed_conv3x3_kernel_fn)( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); + +int lw_packed_conv3x3_stride2_weight_count(uint32_t input_channels, + uint32_t output_channels, + uint64_t* weight_count); +void lw_pack_conv3x3_stride2_weights_f32(const float* weights, uint32_t input_channels, + uint32_t output_channels, float* packed_weights); +void lw_scalar_packed_conv3x3_stride2_pad1_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); +void lw_packed_conv3x3_stride2_pad1_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv_internal.h new file mode 100644 index 00000000..ad93afbb --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_conv_internal.h @@ -0,0 +1,33 @@ +#ifndef LW_PACKED_CONV_INTERNAL_H +#define LW_PACKED_CONV_INTERNAL_H + +/* + * Private packed-weight contract for NCHW 1x1 Conv. LWM keeps canonical OIHW + * weights; a session converts eligible constant weights to [OC/4][IC][4] + * once and reuses them for every execution. + */ + +#include "lw_infer.h" + +#include + +#define LW_PACKED_CONV1X1_OUTPUT_TILE 4u + +typedef void (*lw_packed_conv1x1_kernel_fn)( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); + +int lw_packed_conv1x1_weight_count(uint32_t input_channels, uint32_t output_channels, + uint64_t* weight_count); +void lw_pack_conv1x1_weights_f32(const float* weights, uint32_t input_channels, + uint32_t output_channels, float* packed_weights); + +void lw_scalar_packed_conv1x1_f32(const float* input, const float* packed_weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_packed_conv1x1_f32(const float* input, const float* packed_weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_matmul.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_matmul.c new file mode 100644 index 00000000..75fe5e71 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_matmul.c @@ -0,0 +1,84 @@ +#include "packed_matmul_internal.h" + +#include +#include + +int lw_packed_matmul_weight_count(uint32_t inner_dimension, uint32_t columns, + uint64_t* weight_count) { + uint64_t column_panels; + if (inner_dimension == 0u || columns == 0u || weight_count == NULL) { + return 0; + } + column_panels = ((uint64_t)columns + LW_PACKED_MATMUL_COLUMN_TILE - 1u) / + LW_PACKED_MATMUL_COLUMN_TILE; + if (column_panels > UINT64_MAX / inner_dimension || + column_panels * inner_dimension > UINT64_MAX / LW_PACKED_MATMUL_COLUMN_TILE) { + return 0; + } + *weight_count = column_panels * inner_dimension * LW_PACKED_MATMUL_COLUMN_TILE; + return *weight_count <= (uint64_t)(SIZE_MAX / sizeof(float)); +} + +void lw_pack_matmul_weights_f32(const float* weights, uint32_t inner_dimension, + uint32_t columns, float* packed_weights) { + uint32_t column_panel; + const uint32_t column_panels = + (columns + LW_PACKED_MATMUL_COLUMN_TILE - 1u) / LW_PACKED_MATMUL_COLUMN_TILE; + for (column_panel = 0u; column_panel < column_panels; ++column_panel) { + uint32_t inner; + for (inner = 0u; inner < inner_dimension; ++inner) { + uint32_t lane; + for (lane = 0u; lane < LW_PACKED_MATMUL_COLUMN_TILE; ++lane) { + const uint32_t column = + column_panel * LW_PACKED_MATMUL_COLUMN_TILE + lane; + const uint64_t packed_index = + ((uint64_t)column_panel * inner_dimension + inner) * + LW_PACKED_MATMUL_COLUMN_TILE + + lane; + packed_weights[(size_t)packed_index] = + column < columns + ? weights[(size_t)((uint64_t)inner * columns + column)] + : 0.0f; + } + } + } +} + +void lw_scalar_packed_matmul_shared_f32(const float* input, const float* packed_weights, + float* output, uint32_t batch_count, uint32_t rows, + uint32_t inner_dimension, uint32_t columns) { + uint32_t batch; + const uint32_t column_panels = + (columns + LW_PACKED_MATMUL_COLUMN_TILE - 1u) / LW_PACKED_MATMUL_COLUMN_TILE; + for (batch = 0u; batch < batch_count; ++batch) { + uint32_t row; + for (row = 0u; row < rows; ++row) { + uint32_t column_panel; + for (column_panel = 0u; column_panel < column_panels; ++column_panel) { + float accumulators[LW_PACKED_MATMUL_COLUMN_TILE] = {0.0f}; + const uint32_t column_base = column_panel * LW_PACKED_MATMUL_COLUMN_TILE; + const uint32_t valid_columns = + columns - column_base < LW_PACKED_MATMUL_COLUMN_TILE + ? columns - column_base + : LW_PACKED_MATMUL_COLUMN_TILE; + uint32_t inner; + for (inner = 0u; inner < inner_dimension; ++inner) { + const float input_value = + input[(size_t)(((uint64_t)batch * rows + row) * inner_dimension + inner)]; + const float* packed = + packed_weights + + (size_t)(((uint64_t)column_panel * inner_dimension + inner) * + LW_PACKED_MATMUL_COLUMN_TILE); + uint32_t lane; + for (lane = 0u; lane < valid_columns; ++lane) { + accumulators[lane] += input_value * packed[lane]; + } + } + for (uint32_t lane = 0u; lane < valid_columns; ++lane) { + output[(size_t)(((uint64_t)batch * rows + row) * columns + column_base + + lane)] = accumulators[lane]; + } + } + } + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_matmul_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_matmul_internal.h new file mode 100644 index 00000000..ea6028c2 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/packed_matmul_internal.h @@ -0,0 +1,22 @@ +#ifndef LW_PACKED_MATMUL_INTERNAL_H +#define LW_PACKED_MATMUL_INTERNAL_H + +/* + * Private packed-weight contract for wide shared-weight MatMul. LWM keeps B + * in canonical [K][N] order; eligible x64 AVX2 sessions convert it once to + * [ceil(N/16)][K][16] and reuse the packed copy for every graph execution. + */ + +#include + +#define LW_PACKED_MATMUL_COLUMN_TILE 16u + +int lw_packed_matmul_weight_count(uint32_t inner_dimension, uint32_t columns, + uint64_t* weight_count); +void lw_pack_matmul_weights_f32(const float* weights, uint32_t inner_dimension, + uint32_t columns, float* packed_weights); +void lw_scalar_packed_matmul_shared_f32(const float* input, const float* packed_weights, + float* output, uint32_t batch_count, uint32_t rows, + uint32_t inner_dimension, uint32_t columns); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_activation.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_activation.c new file mode 100644 index 00000000..dfb44a6c --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_activation.c @@ -0,0 +1,94 @@ +#include "scalar_kernels.h" + +/* Portable activation functions used as the reference implementation. */ + +#include +#include +#include + +static lw_status validate_elementwise(const float* input, float* output, uint64_t element_count) { + if (element_count > (uint64_t)(SIZE_MAX / sizeof(float))) { + return LW_STATUS_OUT_OF_BOUNDS; + } + if (element_count != 0u && (input == NULL || output == NULL)) { + return LW_STATUS_INVALID_ARGUMENT; + } + return LW_STATUS_OK; +} + +lw_status lw_scalar_relu_f32(const float* input, float* output, uint64_t element_count) { + uint64_t index; + lw_status status = validate_elementwise(input, output, element_count); + if (status != LW_STATUS_OK) { + return status; + } + for (index = 0u; index < element_count; ++index) { + float value = input[(size_t)index]; + output[(size_t)index] = value > 0.0f ? value : 0.0f; + } + return LW_STATUS_OK; +} + +lw_status lw_scalar_erf_f32(const float* input, float* output, uint64_t element_count) { + uint64_t index; + lw_status status = validate_elementwise(input, output, element_count); + if (status != LW_STATUS_OK) { + return status; + } + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = erff(input[(size_t)index]); + } + return LW_STATUS_OK; +} + +lw_status lw_scalar_hard_sigmoid_f32(const float* input, float* output, uint64_t element_count, + float alpha, float beta) { + uint64_t index; + lw_status status = validate_elementwise(input, output, element_count); + if (status != LW_STATUS_OK) { + return status; + } + if (!isfinite(alpha) || !isfinite(beta)) { + return LW_STATUS_INVALID_ARGUMENT; + } + for (index = 0u; index < element_count; ++index) { + float value = alpha * input[(size_t)index] + beta; + if (value < 0.0f) { + value = 0.0f; + } else if (value > 1.0f) { + value = 1.0f; + } + output[(size_t)index] = value; + } + return LW_STATUS_OK; +} + +lw_status lw_scalar_sigmoid_f32(const float* input, float* output, uint64_t element_count) { + uint64_t index; + lw_status status = validate_elementwise(input, output, element_count); + if (status != LW_STATUS_OK) { + return status; + } + for (index = 0u; index < element_count; ++index) { + float value = input[(size_t)index]; + if (value >= 0.0f) { + output[(size_t)index] = 1.0f / (1.0f + expf(-value)); + } else { + float exponential = expf(value); + output[(size_t)index] = exponential / (1.0f + exponential); + } + } + return LW_STATUS_OK; +} + +lw_status lw_scalar_sqrt_f32(const float* input, float* output, uint64_t element_count) { + uint64_t index; + lw_status status = validate_elementwise(input, output, element_count); + if (status != LW_STATUS_OK) { + return status; + } + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = sqrtf(input[(size_t)index]); + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_conv.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_conv.c new file mode 100644 index 00000000..4a877f86 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_conv.c @@ -0,0 +1,1128 @@ +#include "scalar_kernels.h" + +/* + * Portable convolution implementations plus guarded SIMD dispatch. + * The specialized 1x1, depthwise 3x3 and stride-2 3x3 paths preserve the same + * tensor layout and accumulation order contract as the general fallback. + */ +#include "cpu_features.h" +#include "simd_kernels.h" + +#include +#include +#include +#include + +static lw_status tensor_element_count(uint32_t rank, const int32_t* dimensions, uint64_t* count) { + uint64_t value = 1u; + uint32_t index; + if (rank > LW_MAX_DIMS || (rank != 0u && dimensions == NULL)) { + return LW_STATUS_INVALID_SHAPE; + } + for (index = 0u; index < rank; ++index) { + if (dimensions[index] <= 0 || value > UINT64_MAX / (uint32_t)dimensions[index]) { + return LW_STATUS_INVALID_SHAPE; + } + value *= (uint32_t)dimensions[index]; + } + if (value > (uint64_t)(SIZE_MAX / sizeof(float))) { + return LW_STATUS_OUT_OF_BOUNDS; + } + *count = value; + return LW_STATUS_OK; +} + +static int spatial_output(int32_t input, int32_t kernel, int32_t stride, int32_t dilation, + int32_t pad_before, int32_t pad_after, int32_t* output) { + int64_t effective_kernel; + int64_t numerator; + int64_t value; + if (input <= 0 || kernel <= 0 || stride <= 0 || dilation <= 0 || pad_before < 0 || + pad_after < 0) { + return 0; + } + effective_kernel = (int64_t)(kernel - 1) * dilation + 1; + numerator = (int64_t)input + pad_before + pad_after - effective_kernel; + if (numerator < 0) { + return 0; + } + value = numerator / stride + 1; + if (value <= 0 || value > INT32_MAX) { + return 0; + } + *output = (int32_t)value; + return 1; +} + +static int transposed_spatial_output(int32_t input, int32_t kernel, int32_t stride, + int32_t dilation, int32_t pad_before, int32_t pad_after, + int32_t* output) { + int64_t value; + if (input <= 0 || kernel <= 0 || stride <= 0 || dilation <= 0 || pad_before < 0 || + pad_after < 0) { + return 0; + } + value = (int64_t)(input - 1) * stride - pad_before - pad_after + + (int64_t)(kernel - 1) * dilation + 1; + if (value <= 0 || value > INT32_MAX) { + return 0; + } + *output = (int32_t)value; + return 1; +} + +void lw_scalar_conv1x1_unit_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4], uint32_t groups, + uint32_t input_channels_per_group, + uint32_t output_channels_per_group) { + uint64_t channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + uint32_t group; + for (group = 0u; group < groups; ++group) { + uint32_t input_channel_start = group * input_channels_per_group; + uint32_t output_channel_start = group * output_channels_per_group; + const float* group_input = + input + + (size_t)(((uint64_t)batch * (uint32_t)input_dimensions[1] + input_channel_start) * + channel_plane); + float* group_output = + output + + (size_t)(((uint64_t)batch * (uint32_t)output_dimensions[1] + output_channel_start) * + channel_plane); + uint32_t output_channel_in_group; + for (output_channel_in_group = 0u; output_channel_in_group < output_channels_per_group; + ++output_channel_in_group) { + uint32_t output_channel = output_channel_start + output_channel_in_group; + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * input_channels_per_group); + float* output_channel_data = + group_output + (size_t)((uint64_t)output_channel_in_group * channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + uint64_t spatial; + uint32_t input_channel_in_group; + for (spatial = 0u; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel_in_group = 0u; input_channel_in_group < input_channels_per_group; + ++input_channel_in_group) { + const float* input_channel_data = + group_input + (size_t)((uint64_t)input_channel_in_group * channel_plane); + float weight = output_channel_weights[input_channel_in_group]; + for (spatial = 0u; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] += + input_channel_data[(size_t)spatial] * weight; + } + } + } + } + } +} + +void lw_scalar_depthwise_conv3x3_unit_pad1_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]) { + uint32_t height = (uint32_t)dimensions[2]; + uint32_t width = (uint32_t)dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)dimensions[0]; ++batch) { + uint32_t channel; + for (channel = 0u; channel < (uint32_t)dimensions[1]; ++channel) { + uint64_t channel_offset = + ((uint64_t)batch * (uint32_t)dimensions[1] + channel) * channel_plane; + const float* input_channel = input + (size_t)channel_offset; + const float* channel_weights = weights + (size_t)channel * 9u; + float* output_channel = output + (size_t)channel_offset; + float initial = bias == NULL ? 0.0f : bias[channel]; + uint64_t spatial; + uint32_t kernel_y; + for (spatial = 0u; spatial < channel_plane; ++spatial) { + output_channel[(size_t)spatial] = initial; + } + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = kernel_y == 2u && height != 0u ? height - 1u : height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = kernel_x == 2u && width != 0u ? width - 1u : width; + float weight = channel_weights[kernel_y * 3u + kernel_x]; + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y + kernel_y - 1u; + const float* input_row = + input_channel + (size_t)((uint64_t)input_y * width); + float* output_row = output_channel + (size_t)((uint64_t)output_y * width); + uint32_t output_x; + for (output_x = output_x_begin; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 1u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } +} + +void lw_scalar_depthwise_conv3x3_stride2x1_pad1_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { + uint32_t input_height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint32_t output_height = (uint32_t)output_dimensions[2]; + uint64_t input_channel_plane = (uint64_t)input_height * width; + uint64_t output_channel_plane = (uint64_t)output_height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + uint32_t channel; + for (channel = 0u; channel < (uint32_t)input_dimensions[1]; ++channel) { + uint64_t input_channel_offset = + ((uint64_t)batch * (uint32_t)input_dimensions[1] + channel) * input_channel_plane; + uint64_t output_channel_offset = + ((uint64_t)batch * (uint32_t)output_dimensions[1] + channel) * + output_channel_plane; + const float* input_channel = input + (size_t)input_channel_offset; + const float* channel_weights = weights + (size_t)channel * 9u; + float* output_channel = output + (size_t)output_channel_offset; + float initial = bias == NULL ? 0.0f : bias[channel]; + uint64_t spatial; + uint32_t kernel_y; + for (spatial = 0u; spatial < output_channel_plane; ++spatial) { + output_channel[(size_t)spatial] = initial; + } + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = kernel_y == 2u ? input_height / 2u : output_height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = kernel_x == 2u && width != 0u ? width - 1u : width; + float weight = channel_weights[kernel_y * 3u + kernel_x]; + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y * 2u + kernel_y - 1u; + const float* input_row = + input_channel + (size_t)((uint64_t)input_y * width); + float* output_row = output_channel + (size_t)((uint64_t)output_y * width); + uint32_t output_x; + for (output_x = output_x_begin; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 1u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } +} + +void lw_scalar_depthwise_conv5x5_unit_pad2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]) { + uint32_t height = (uint32_t)dimensions[2]; + uint32_t width = (uint32_t)dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)dimensions[0]; ++batch) { + uint32_t channel; + for (channel = 0u; channel < (uint32_t)dimensions[1]; ++channel) { + uint64_t channel_offset = + ((uint64_t)batch * (uint32_t)dimensions[1] + channel) * channel_plane; + const float* input_channel = input + (size_t)channel_offset; + const float* channel_weights = weights + (size_t)channel * 25u; + float* output_channel = output + (size_t)channel_offset; + float initial = bias == NULL ? 0.0f : bias[channel]; + uint64_t spatial; + uint32_t kernel_y; + for (spatial = 0u; spatial < channel_plane; ++spatial) { + output_channel[(size_t)spatial] = initial; + } + for (kernel_y = 0u; kernel_y < 5u; ++kernel_y) { + uint32_t output_y_begin = kernel_y < 2u ? 2u - kernel_y : 0u; + uint32_t output_y_trim = kernel_y > 2u ? kernel_y - 2u : 0u; + uint32_t output_y_end = output_y_trim < height ? height - output_y_trim : 0u; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 5u; ++kernel_x) { + uint32_t output_x_begin = kernel_x < 2u ? 2u - kernel_x : 0u; + uint32_t output_x_trim = kernel_x > 2u ? kernel_x - 2u : 0u; + uint32_t output_x_end = output_x_trim < width ? width - output_x_trim : 0u; + float weight = channel_weights[kernel_y * 5u + kernel_x]; + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y + kernel_y - 2u; + const float* input_row = + input_channel + (size_t)((uint64_t)input_y * width); + float* output_row = output_channel + (size_t)((uint64_t)output_y * width); + uint32_t output_x; + for (output_x = output_x_begin; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 2u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } +} + +void lw_scalar_conv3x3_unit_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + uint32_t height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * (uint32_t)input_dimensions[1] * 9u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + uint64_t spatial; + uint32_t input_channel; + for (spatial = 0u; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = kernel_y == 2u ? height - 1u : height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = kernel_x == 2u ? width - 1u : width; + float weight = weight_channel_data[kernel_y * 3u + kernel_x]; + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y + kernel_y - 1u; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x; + for (output_x = output_x_begin; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 1u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } + } +} + +void lw_scalar_conv2x2_unit_pad_end1_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + uint32_t height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * (uint32_t)input_dimensions[1] * 4u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + uint64_t spatial; + uint32_t input_channel; + for (spatial = 0u; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 4u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 2u; ++kernel_y) { + uint32_t output_y_end = height - kernel_y; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 2u; ++kernel_x) { + uint32_t output_x_end = width - kernel_x; + float weight = weight_channel_data[kernel_y * 2u + kernel_x]; + uint32_t output_y; + for (output_y = 0u; output_y < output_y_end; ++output_y) { + const float* input_row = + input_channel_data + + (size_t)((uint64_t)(output_y + kernel_y) * width + kernel_x); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x; + for (output_x = 0u; output_x < output_x_end; ++output_x) { + output_row[output_x] += input_row[output_x] * weight; + } + } + } + } + } + } + } +} + +void lw_scalar_conv3x3_stride2_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + uint64_t input_channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + uint64_t output_channel_plane = + (uint64_t)(uint32_t)output_dimensions[2] * (uint32_t)output_dimensions[3]; + uint32_t output_y_begin[3]; + uint32_t output_y_end[3]; + uint32_t output_x_begin[3]; + uint32_t output_x_end[3]; + uint32_t kernel_index; + uint32_t batch; + for (kernel_index = 0u; kernel_index < 3u; ++kernel_index) { + output_y_begin[kernel_index] = 0u; + output_y_end[kernel_index] = (uint32_t)output_dimensions[2]; + while (output_y_begin[kernel_index] < output_y_end[kernel_index] && + (int64_t)output_y_begin[kernel_index] * 2 - 1 + kernel_index < 0) { + ++output_y_begin[kernel_index]; + } + while (output_y_end[kernel_index] > output_y_begin[kernel_index] && + (int64_t)(output_y_end[kernel_index] - 1u) * 2 - 1 + kernel_index >= + input_dimensions[2]) { + --output_y_end[kernel_index]; + } + output_x_begin[kernel_index] = 0u; + output_x_end[kernel_index] = (uint32_t)output_dimensions[3]; + while (output_x_begin[kernel_index] < output_x_end[kernel_index] && + (int64_t)output_x_begin[kernel_index] * 2 - 1 + kernel_index < 0) { + ++output_x_begin[kernel_index]; + } + while (output_x_end[kernel_index] > output_x_begin[kernel_index] && + (int64_t)(output_x_end[kernel_index] - 1u) * 2 - 1 + kernel_index >= + input_dimensions[3]) { + --output_x_end[kernel_index]; + } + } + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * input_channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * + output_channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * (uint32_t)input_dimensions[1] * 9u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * output_channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + uint64_t spatial; + uint32_t input_channel; + for (spatial = 0u; spatial < output_channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * input_channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_y; + float weight = weight_channel_data[(size_t)kernel_y * 3u + kernel_x]; + for (output_y = output_y_begin[kernel_y]; output_y < output_y_end[kernel_y]; + ++output_y) { + uint32_t input_y = (uint32_t)((int64_t)output_y * 2 - 1 + kernel_y); + const float* input_row = + input_channel_data + + (size_t)((uint64_t)input_y * (uint32_t)input_dimensions[3]); + float* output_row = + output_channel_data + + (size_t)((uint64_t)output_y * (uint32_t)output_dimensions[3]); + uint32_t output_x; + for (output_x = output_x_begin[kernel_x]; + output_x < output_x_end[kernel_x]; ++output_x) { + uint32_t input_x = (uint32_t)((int64_t)output_x * 2 - 1 + kernel_x); + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } + } +} + +lw_status lw_scalar_conv2d_f32(const float* input, const float* weights, const float* bias, + uint32_t bias_count, float* output, + const int32_t input_dimensions[4], + const int32_t weight_dimensions[4], + const int32_t output_dimensions[4], const int32_t kernel[2], + const int32_t strides[2], const int32_t dilations[2], + const int32_t pads[4], uint32_t groups) { + uint64_t input_count; + uint64_t weight_count; + uint64_t output_count; + int32_t expected_height; + int32_t expected_width; + uint32_t input_channels_per_group; + uint32_t output_channels_per_group; + uint32_t batch; + uint32_t group; + lw_status status; + if (input == NULL || weights == NULL || output == NULL || input_dimensions == NULL || + weight_dimensions == NULL || output_dimensions == NULL || kernel == NULL || + strides == NULL || dilations == NULL || pads == NULL || output == input || + output == weights || (bias != NULL && output == bias)) { + return LW_STATUS_INVALID_ARGUMENT; + } + if ((bias == NULL && bias_count != 0u) || (bias != NULL && bias_count == 0u) || groups == 0u) { + return LW_STATUS_INVALID_SHAPE; + } + status = tensor_element_count(4u, input_dimensions, &input_count); + if (status != LW_STATUS_OK) { + return status; + } + status = tensor_element_count(4u, weight_dimensions, &weight_count); + if (status != LW_STATUS_OK) { + return status; + } + status = tensor_element_count(4u, output_dimensions, &output_count); + if (status != LW_STATUS_OK) { + return status; + } + (void)input_count; + (void)weight_count; + (void)output_count; + if (weight_dimensions[0] != output_dimensions[1] || + output_dimensions[0] != input_dimensions[0] || + (uint64_t)(uint32_t)weight_dimensions[1] * groups != (uint32_t)input_dimensions[1] || + (uint32_t)input_dimensions[1] % groups != 0u || + (uint32_t)output_dimensions[1] % groups != 0u || weight_dimensions[2] != kernel[0] || + weight_dimensions[3] != kernel[1] || + (bias != NULL && bias_count != (uint32_t)output_dimensions[1]) || + !spatial_output(input_dimensions[2], kernel[0], strides[0], dilations[0], pads[0], pads[2], + &expected_height) || + !spatial_output(input_dimensions[3], kernel[1], strides[1], dilations[1], pads[1], pads[3], + &expected_width) || + output_dimensions[2] != expected_height || output_dimensions[3] != expected_width) { + return LW_STATUS_INVALID_SHAPE; + } + input_channels_per_group = (uint32_t)input_dimensions[1] / groups; + output_channels_per_group = (uint32_t)output_dimensions[1] / groups; + + /* Match common PP-OCR convolution shapes before entering the general + * nested loops. Every branch uses the same validated NCHW contract. */ + if (kernel[0] == 1 && kernel[1] == 1 && strides[0] == 1 && strides[1] == 1 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 0 && pads[1] == 0 && pads[2] == 0 && + pads[3] == 0) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_conv1x1_unit_f32(input, weights, bias, output, input_dimensions, + output_dimensions, groups, input_channels_per_group, + output_channels_per_group); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_conv1x1_unit_f32(input, weights, bias, output, input_dimensions, + output_dimensions, groups, input_channels_per_group, + output_channels_per_group); + } else { + lw_scalar_conv1x1_unit_f32(input, weights, bias, output, input_dimensions, + output_dimensions, groups, input_channels_per_group, + output_channels_per_group); + } + return LW_STATUS_OK; + } + if (groups == (uint32_t)input_dimensions[1] && output_dimensions[1] == input_dimensions[1] && + weight_dimensions[1] == 1 && kernel[0] == 3 && kernel[1] == 3 && strides[0] == 1 && + strides[1] == 1 && dilations[0] == 1 && dilations[1] == 1 && pads[0] == 1 && pads[1] == 1 && + pads[2] == 1 && pads[3] == 1 && output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_depthwise_conv3x3_unit_pad1_f32(input, weights, bias, output, input_dimensions); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_depthwise_conv3x3_unit_pad1_f32(input, weights, bias, output, input_dimensions); + } else { + lw_scalar_depthwise_conv3x3_unit_pad1_f32(input, weights, bias, output, + input_dimensions); + } + return LW_STATUS_OK; + } + if (groups == (uint32_t)input_dimensions[1] && output_dimensions[1] == input_dimensions[1] && + weight_dimensions[1] == 1 && kernel[0] == 3 && kernel[1] == 3 && strides[0] == 2 && + strides[1] == 1 && dilations[0] == 1 && dilations[1] == 1 && pads[0] == 1 && pads[1] == 1 && + pads[2] == 1 && pads[3] == 1 && output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_depthwise_conv3x3_stride2x1_pad1_f32( + input, weights, bias, output, input_dimensions, output_dimensions); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_depthwise_conv3x3_stride2x1_pad1_f32( + input, weights, bias, output, input_dimensions, output_dimensions); + } else { + lw_scalar_depthwise_conv3x3_stride2x1_pad1_f32( + input, weights, bias, output, input_dimensions, output_dimensions); + } + return LW_STATUS_OK; + } + if (groups == (uint32_t)input_dimensions[1] && output_dimensions[1] == input_dimensions[1] && + weight_dimensions[1] == 1 && kernel[0] == 5 && kernel[1] == 5 && strides[0] == 1 && + strides[1] == 1 && dilations[0] == 1 && dilations[1] == 1 && pads[0] == 2 && pads[1] == 2 && + pads[2] == 2 && pads[3] == 2 && output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_depthwise_conv5x5_unit_pad2_f32(input, weights, bias, output, input_dimensions); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_depthwise_conv5x5_unit_pad2_f32(input, weights, bias, output, input_dimensions); + } else { + lw_scalar_depthwise_conv5x5_unit_pad2_f32(input, weights, bias, output, + input_dimensions); + } + return LW_STATUS_OK; + } + if (groups == (uint32_t)input_dimensions[1] && output_dimensions[1] == input_dimensions[1] && + weight_dimensions[1] == 1 && kernel[0] == 9 && kernel[1] == 9 && strides[0] == 1 && + strides[1] == 1 && dilations[0] == 1 && dilations[1] == 1 && pads[0] == 4 && pads[1] == 4 && + pads[2] == 4 && pads[3] == 4 && input_dimensions[2] >= 9 && input_dimensions[3] >= 9 && + output_dimensions[2] == input_dimensions[2] && output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_depthwise_conv9x9_unit_pad4_f32(input, weights, bias, output, input_dimensions); + } else { + goto general_convolution; + } + return LW_STATUS_OK; + } + if (groups == 1u && kernel[0] == 7 && kernel[1] == 1 && strides[0] == 1 && strides[1] == 1 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 3 && pads[1] == 0 && pads[2] == 3 && + pads[3] == 0 && input_dimensions[2] >= 7 && output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_conv7x1_unit_pad3_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else { + goto general_convolution; + } + return LW_STATUS_OK; + } + if (groups == 1u && kernel[0] == 1 && kernel[1] == 7 && strides[0] == 1 && strides[1] == 1 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 0 && pads[1] == 3 && pads[2] == 0 && + pads[3] == 3 && input_dimensions[3] >= 7 && output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_conv1x7_unit_pad3_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else { + goto general_convolution; + } + return LW_STATUS_OK; + } + if (groups == 1u && kernel[0] == 5 && kernel[1] == 1 && strides[0] == 1 && strides[1] == 1 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 2 && pads[1] == 0 && pads[2] == 2 && + pads[3] == 0 && input_dimensions[2] >= 5 && output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_conv5x1_unit_pad2_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else { + goto general_convolution; + } + return LW_STATUS_OK; + } + if (groups == 1u && kernel[0] == 1 && kernel[1] == 5 && strides[0] == 1 && strides[1] == 1 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 0 && pads[1] == 2 && pads[2] == 0 && + pads[3] == 2 && input_dimensions[3] >= 5 && output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_conv1x5_unit_pad2_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else { + goto general_convolution; + } + return LW_STATUS_OK; + } + if (groups == 1u && kernel[0] == 3 && kernel[1] == 3 && strides[0] == 1 && strides[1] == 1 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 1 && pads[1] == 1 && pads[2] == 1 && + pads[3] == 1 && output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_conv3x3_unit_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else if (lw_simd_level_is_neon(simd_level)) { + lw_neon_conv3x3_unit_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_conv3x3_unit_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else { + lw_scalar_conv3x3_unit_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } + return LW_STATUS_OK; + } + if (groups == 1u && kernel[0] == 7 && kernel[1] == 7 && strides[0] == 1 && strides[1] == 1 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 3 && pads[1] == 3 && pads[2] == 3 && + pads[3] == 3 && input_dimensions[2] >= 7 && input_dimensions[3] >= 7 && + output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + if (((uint32_t)output_dimensions[1] & 3u) == 0u) { + lw_avx2_conv7x7_four_outputs_unit_pad3_f32( + input, weights, bias, output, input_dimensions, output_dimensions); + } else { + lw_avx2_conv7x7_unit_pad3_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } + } else { + /* Keep the generic reference path for ARM, WASM, and scalar hosts. */ + goto general_convolution; + } + return LW_STATUS_OK; + } + if (groups == 1u && kernel[0] == 5 && kernel[1] == 5 && strides[0] == 1 && strides[1] == 1 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 2 && pads[1] == 2 && pads[2] == 2 && + pads[3] == 2 && input_dimensions[2] >= 5 && input_dimensions[3] >= 5 && + output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + if (((uint32_t)output_dimensions[1] & 3u) == 0u) { + lw_avx2_conv5x5_four_outputs_unit_pad2_f32( + input, weights, bias, output, input_dimensions, output_dimensions); + } else { + lw_avx2_conv5x5_unit_pad2_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } + } else { + /* Keep the generic reference path for ARM, WASM, and scalar hosts. */ + goto general_convolution; + } + return LW_STATUS_OK; + } + if (groups == 1u && kernel[0] == 2 && kernel[1] == 2 && strides[0] == 1 && strides[1] == 1 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 0 && pads[1] == 0 && pads[2] == 1 && + pads[3] == 1 && output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_conv2x2_unit_pad_end1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_conv2x2_unit_pad_end1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else { + lw_scalar_conv2x2_unit_pad_end1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } + return LW_STATUS_OK; + } + if (groups == 1u && kernel[0] == 3 && kernel[1] == 3 && strides[0] == 2 && strides[1] == 2 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 1 && pads[1] == 1 && pads[2] == 1 && + pads[3] == 1) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_conv3x3_stride2_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_conv3x3_stride2_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else { + lw_scalar_conv3x3_stride2_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } + return LW_STATUS_OK; + } +general_convolution: + /* General grouped-convolution fallback. Kernel bounds are trimmed once per + * output coordinate, avoiding an inner-loop branch for padded pixels. */ + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + for (group = 0u; group < groups; ++group) { + uint32_t output_channel_in_group; + uint32_t input_channel_start = group * input_channels_per_group; + uint32_t output_channel_start = group * output_channels_per_group; + uint64_t input_channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + uint64_t output_channel_plane = + (uint64_t)(uint32_t)expected_height * (uint32_t)expected_width; + uint64_t kernel_channel_plane = (uint64_t)(uint32_t)kernel[0] * (uint32_t)kernel[1]; + const float* group_input = + input + + (size_t)(((uint64_t)batch * (uint32_t)input_dimensions[1] + input_channel_start) * + input_channel_plane); + float* group_output = + output + + (size_t)(((uint64_t)batch * (uint32_t)output_dimensions[1] + output_channel_start) * + output_channel_plane); + for (output_channel_in_group = 0u; output_channel_in_group < output_channels_per_group; + ++output_channel_in_group) { + uint32_t output_channel = output_channel_start + output_channel_in_group; + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * input_channels_per_group * + kernel_channel_plane); + float* output_channel_data = + group_output + + (size_t)((uint64_t)output_channel_in_group * output_channel_plane); + uint32_t output_y; + for (output_y = 0u; output_y < (uint32_t)expected_height; ++output_y) { + int64_t input_y_start = (int64_t)output_y * strides[0] - pads[0]; + uint32_t kernel_y_begin = 0u; + uint32_t kernel_y_end = (uint32_t)kernel[0]; + uint32_t output_x; + while (kernel_y_begin < kernel_y_end && + input_y_start + (int64_t)kernel_y_begin * dilations[0] < 0) { + ++kernel_y_begin; + } + while (kernel_y_end > kernel_y_begin && + input_y_start + (int64_t)(kernel_y_end - 1u) * dilations[0] >= + input_dimensions[2]) { + --kernel_y_end; + } + for (output_x = 0u; output_x < (uint32_t)expected_width; ++output_x) { + int64_t input_x_start = (int64_t)output_x * strides[1] - pads[1]; + uint32_t kernel_x_begin = 0u; + uint32_t kernel_x_end = (uint32_t)kernel[1]; + float sum = bias == NULL ? 0.0f : bias[output_channel]; + uint32_t input_channel_in_group; + while (kernel_x_begin < kernel_x_end && + input_x_start + (int64_t)kernel_x_begin * dilations[1] < 0) { + ++kernel_x_begin; + } + while (kernel_x_end > kernel_x_begin && + input_x_start + (int64_t)(kernel_x_end - 1u) * dilations[1] >= + input_dimensions[3]) { + --kernel_x_end; + } + for (input_channel_in_group = 0u; + input_channel_in_group < input_channels_per_group; + ++input_channel_in_group) { + const float* input_channel_data = + group_input + + (size_t)((uint64_t)input_channel_in_group * input_channel_plane); + const float* weight_channel_data = + output_channel_weights + + (size_t)((uint64_t)input_channel_in_group * kernel_channel_plane); + uint32_t kernel_y; + for (kernel_y = kernel_y_begin; kernel_y < kernel_y_end; ++kernel_y) { + uint32_t input_y = + (uint32_t)(input_y_start + (int64_t)kernel_y * dilations[0]); + const float* input_row = + input_channel_data + + (size_t)((uint64_t)input_y * (uint32_t)input_dimensions[3]); + const float* weight_row = + weight_channel_data + + (size_t)((uint64_t)kernel_y * (uint32_t)kernel[1]); + uint32_t kernel_x; + for (kernel_x = kernel_x_begin; kernel_x < kernel_x_end; + ++kernel_x) { + uint32_t input_x = (uint32_t)(input_x_start + + (int64_t)kernel_x * dilations[1]); + sum += input_row[(size_t)input_x] * weight_row[kernel_x]; + } + } + } + output_channel_data[(size_t)((uint64_t)output_y * (uint32_t)expected_width + + output_x)] = sum; + } + } + } + } + } + return LW_STATUS_OK; +} + +void lw_scalar_conv_transpose2x2_stride2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + lw_scalar_conv_transpose2x2_stride2_range_f32( + input, weights, bias, output, input_dimensions, output_dimensions, 0u, + (uint32_t)output_dimensions[1]); +} + +void lw_scalar_conv_transpose2x2_stride2_range_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4], + uint32_t output_channel_begin, uint32_t output_channel_end) { + uint32_t input_channels = (uint32_t)input_dimensions[1]; + uint32_t output_channels = (uint32_t)output_dimensions[1]; + uint32_t input_height = (uint32_t)input_dimensions[2]; + uint32_t input_width = (uint32_t)input_dimensions[3]; + uint32_t output_width = (uint32_t)output_dimensions[3]; + uint64_t input_plane = (uint64_t)input_height * input_width; + uint64_t output_plane = (uint64_t)(uint32_t)output_dimensions[2] * output_width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = input + (size_t)((uint64_t)batch * input_channels * input_plane); + float* batch_output = output + (size_t)((uint64_t)batch * output_channels * output_plane); + uint32_t output_channel; + for (output_channel = output_channel_begin; output_channel < output_channel_end; + ++output_channel) { + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * output_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + uint64_t output_index; + uint32_t input_channel; + for (output_index = 0u; output_index < output_plane; ++output_index) { + output_channel_data[(size_t)output_index] = initial; + } + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * input_plane); + const float* channel_weights = + weights + + (size_t)(((uint64_t)input_channel * output_channels + output_channel) * 4u); + uint32_t input_y; + for (input_y = 0u; input_y < input_height; ++input_y) { + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * input_width); + float* output_row0 = + output_channel_data + (size_t)((uint64_t)(input_y * 2u) * output_width); + float* output_row1 = output_row0 + output_width; + uint32_t input_x; + for (input_x = 0u; input_x < input_width; ++input_x) { + float value = input_row[input_x]; + uint32_t output_x = input_x * 2u; + output_row0[output_x] += value * channel_weights[0]; + output_row0[output_x + 1u] += value * channel_weights[1]; + output_row1[output_x] += value * channel_weights[2]; + output_row1[output_x + 1u] += value * channel_weights[3]; + } + } + } + } + } +} + +lw_status lw_scalar_conv_transpose2d_f32( + const float* input, const float* weights, const float* bias, uint32_t bias_count, float* output, + const int32_t input_dimensions[4], const int32_t weight_dimensions[4], + const int32_t output_dimensions[4], const int32_t kernel[2], const int32_t strides[2], + const int32_t dilations[2], const int32_t pads[4], uint32_t groups) { + uint64_t input_count; + uint64_t weight_count; + uint64_t output_count; + int32_t expected_height; + int32_t expected_width; + uint32_t input_channels_per_group; + uint32_t output_channels_per_group; + uint32_t batch; + lw_status status; + if (input == NULL || weights == NULL || output == NULL || input == output || + weights == output || (bias != NULL && bias == output) || input_dimensions == NULL || + weight_dimensions == NULL || output_dimensions == NULL || kernel == NULL || + strides == NULL || dilations == NULL || pads == NULL || groups == 0u) { + return LW_STATUS_INVALID_ARGUMENT; + } + status = tensor_element_count(4u, input_dimensions, &input_count); + if (status != LW_STATUS_OK) { + return status; + } + status = tensor_element_count(4u, weight_dimensions, &weight_count); + if (status != LW_STATUS_OK) { + return status; + } + status = tensor_element_count(4u, output_dimensions, &output_count); + if (status != LW_STATUS_OK) { + return status; + } + (void)input_count; + (void)weight_count; + if (groups > (uint32_t)input_dimensions[1] || (uint32_t)input_dimensions[1] % groups != 0u || + weight_dimensions[0] != input_dimensions[1] || weight_dimensions[2] != kernel[0] || + weight_dimensions[3] != kernel[1] || output_dimensions[0] != input_dimensions[0] || + (uint64_t)(uint32_t)weight_dimensions[1] * groups != (uint32_t)output_dimensions[1] || + (bias == NULL ? bias_count != 0u : bias_count != (uint32_t)output_dimensions[1]) || + !transposed_spatial_output(input_dimensions[2], kernel[0], strides[0], dilations[0], + pads[0], pads[2], &expected_height) || + !transposed_spatial_output(input_dimensions[3], kernel[1], strides[1], dilations[1], + pads[1], pads[3], &expected_width) || + output_dimensions[2] != expected_height || output_dimensions[3] != expected_width) { + return LW_STATUS_INVALID_SHAPE; + } + input_channels_per_group = (uint32_t)input_dimensions[1] / groups; + output_channels_per_group = (uint32_t)weight_dimensions[1]; + if (groups == 1u && kernel[0] == 2 && kernel[1] == 2 && strides[0] == 2 && strides[1] == 2 && + dilations[0] == 1 && dilations[1] == 1 && pads[0] == 0 && pads[1] == 0 && pads[2] == 0 && + pads[3] == 0) { + lw_simd_level simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_conv_transpose2x2_stride2_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else if (lw_simd_level_is_neon(simd_level)) { + lw_neon_conv_transpose2x2_stride2_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_conv_transpose2x2_stride2_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + } else { + lw_scalar_conv_transpose2x2_stride2_f32(input, weights, bias, output, + input_dimensions, output_dimensions); + } + return LW_STATUS_OK; + } + for (batch = 0u; batch < (uint32_t)output_dimensions[0]; ++batch) { + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + float initial = bias == NULL ? 0.0f : bias[output_channel]; + uint64_t plane = (uint64_t)(uint32_t)expected_height * (uint32_t)expected_width; + uint64_t base = + ((uint64_t)batch * (uint32_t)output_dimensions[1] + output_channel) * plane; + uint64_t index; + for (index = 0u; index < plane; ++index) { + output[(size_t)(base + index)] = initial; + } + } + } + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + uint32_t group; + for (group = 0u; group < groups; ++group) { + uint32_t input_channel_in_group; + for (input_channel_in_group = 0u; input_channel_in_group < input_channels_per_group; + ++input_channel_in_group) { + uint32_t input_channel = group * input_channels_per_group + input_channel_in_group; + uint32_t input_y; + for (input_y = 0u; input_y < (uint32_t)input_dimensions[2]; ++input_y) { + uint32_t input_x; + for (input_x = 0u; input_x < (uint32_t)input_dimensions[3]; ++input_x) { + uint64_t input_offset = + (((uint64_t)batch * (uint32_t)input_dimensions[1] + input_channel) * + (uint32_t)input_dimensions[2] + + input_y) * + (uint32_t)input_dimensions[3] + + input_x; + float input_value = input[(size_t)input_offset]; + uint32_t output_channel_in_group; + for (output_channel_in_group = 0u; + output_channel_in_group < output_channels_per_group; + ++output_channel_in_group) { + uint32_t output_channel = + group * output_channels_per_group + output_channel_in_group; + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < (uint32_t)kernel[0]; ++kernel_y) { + int64_t output_y = (int64_t)input_y * strides[0] - pads[0] + + (int64_t)kernel_y * dilations[0]; + uint32_t kernel_x; + if (output_y < 0 || output_y >= expected_height) { + continue; + } + for (kernel_x = 0u; kernel_x < (uint32_t)kernel[1]; ++kernel_x) { + int64_t output_x = (int64_t)input_x * strides[1] - pads[1] + + (int64_t)kernel_x * dilations[1]; + uint64_t weight_offset; + uint64_t output_offset; + if (output_x < 0 || output_x >= expected_width) { + continue; + } + weight_offset = + (((uint64_t)input_channel * output_channels_per_group + + output_channel_in_group) * + (uint32_t)kernel[0] + + kernel_y) * + (uint32_t)kernel[1] + + kernel_x; + output_offset = + (((uint64_t)batch * (uint32_t)output_dimensions[1] + + output_channel) * + (uint32_t)expected_height + + (uint32_t)output_y) * + (uint32_t)expected_width + + (uint32_t)output_x; + output[(size_t)output_offset] += + input_value * weights[(size_t)weight_offset]; + } + } + } + } + } + } + } + } + (void)output_count; + return LW_STATUS_OK; +} + +lw_status lw_scalar_batch_normalization_f32(const float* input, const float* scale, + const float* bias, const float* mean, + const float* variance, uint32_t parameter_count, + float epsilon, float* output, uint32_t rank, + const int32_t* dimensions) { + uint64_t element_count; + uint64_t spatial_count = 1u; + uint32_t channel_count; + uint32_t channel; + lw_status status; + if (input == NULL || scale == NULL || bias == NULL || mean == NULL || variance == NULL || + output == NULL || output == scale || output == bias || output == mean || + output == variance) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (rank < 2u || rank > LW_MAX_DIMS || dimensions == NULL || !isfinite(epsilon) || + epsilon <= 0.0f) { + return LW_STATUS_INVALID_SHAPE; + } + status = tensor_element_count(rank, dimensions, &element_count); + if (status != LW_STATUS_OK) { + return status; + } + channel_count = (uint32_t)dimensions[1]; + if (parameter_count != channel_count) { + return LW_STATUS_INVALID_SHAPE; + } + for (channel = 2u; channel < rank; ++channel) { + spatial_count *= (uint32_t)dimensions[channel]; + } + for (channel = 0u; channel < channel_count; ++channel) { + float denominator = variance[channel] + epsilon; + if (!isfinite(scale[channel]) || !isfinite(bias[channel]) || !isfinite(mean[channel]) || + !isfinite(variance[channel]) || !isfinite(denominator) || denominator <= 0.0f) { + return LW_STATUS_INVALID_ARGUMENT; + } + } + for (channel = 0u; channel < channel_count; ++channel) { + float factor = scale[channel] / sqrtf(variance[channel] + epsilon); + uint32_t batch; + for (batch = 0u; batch < (uint32_t)dimensions[0]; ++batch) { + uint64_t offset = ((uint64_t)batch * channel_count + channel) * spatial_count; + uint64_t spatial; + for (spatial = 0u; spatial < spatial_count; ++spatial) { + float value = input[(size_t)(offset + spatial)]; + output[(size_t)(offset + spatial)] = + (value - mean[channel]) * factor + bias[channel]; + } + } + } + (void)element_count; + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_elementwise.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_elementwise.c new file mode 100644 index 00000000..e8842d4a --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_elementwise.c @@ -0,0 +1,292 @@ +#include "scalar_kernels.h" + +/* Elementwise arithmetic with NumPy-style broadcasting and SIMD fast paths. */ +#include "cpu_features.h" +#include "simd_kernels.h" + +#include +#include +#include + +static lw_status make_contiguous_strides(uint32_t rank, const int32_t* dimensions, + uint64_t* strides, uint64_t* element_count) { + uint64_t count = 1u; + uint32_t axis; + if (rank > LW_MAX_DIMS || (rank != 0u && dimensions == NULL)) { + return LW_STATUS_INVALID_SHAPE; + } + for (axis = rank; axis > 0u; --axis) { + int32_t dimension = dimensions[axis - 1u]; + if (dimension <= 0 || count > UINT64_MAX / (uint32_t)dimension) { + return LW_STATUS_INVALID_SHAPE; + } + strides[axis - 1u] = count; + count *= (uint32_t)dimension; + } + if (count > (uint64_t)(SIZE_MAX / sizeof(float))) { + return LW_STATUS_OUT_OF_BOUNDS; + } + *element_count = count; + return LW_STATUS_OK; +} + +void lw_scalar_binary_contiguous_f32(lw_scalar_binary_op operation, const float* left, + const float* right, float* output, uint64_t element_count) { + uint64_t index; + if (operation == LW_SCALAR_BINARY_ADD) { + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = left[(size_t)index] + right[(size_t)index]; + } + } else if (operation == LW_SCALAR_BINARY_MUL) { + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = left[(size_t)index] * right[(size_t)index]; + } + } else if (operation == LW_SCALAR_BINARY_DIV) { + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = left[(size_t)index] / right[(size_t)index]; + } + } else if (operation == LW_SCALAR_BINARY_SUB) { + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = left[(size_t)index] - right[(size_t)index]; + } + } else { + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = powf(left[(size_t)index], right[(size_t)index]); + } + } +} + +void lw_scalar_binary_right_scalar_f32(lw_scalar_binary_op operation, const float* left, + float right, float* output, uint64_t element_count) { + uint64_t index; + if (operation == LW_SCALAR_BINARY_ADD) { + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = left[(size_t)index] + right; + } + } else if (operation == LW_SCALAR_BINARY_MUL) { + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = left[(size_t)index] * right; + } + } else if (operation == LW_SCALAR_BINARY_DIV) { + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = left[(size_t)index] / right; + } + } else if (operation == LW_SCALAR_BINARY_SUB) { + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = left[(size_t)index] - right; + } + } else { + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = powf(left[(size_t)index], right); + } + } +} + +static void dispatch_binary_contiguous_for_level_f32(lw_scalar_binary_op operation, + const float* left, const float* right, + float* output, uint64_t element_count, + lw_simd_level simd_level) { + if (operation != LW_SCALAR_BINARY_ADD && operation != LW_SCALAR_BINARY_MUL && + operation != LW_SCALAR_BINARY_DIV) { + lw_scalar_binary_contiguous_f32(operation, left, right, output, element_count); + } else if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_binary_contiguous_f32(operation, left, right, output, element_count); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_binary_contiguous_f32(operation, left, right, output, element_count); + } else { + lw_scalar_binary_contiguous_f32(operation, left, right, output, element_count); + } +} + +static void dispatch_binary_contiguous_f32(lw_scalar_binary_op operation, const float* left, + const float* right, float* output, + uint64_t element_count) { + dispatch_binary_contiguous_for_level_f32(operation, left, right, output, element_count, + lw_detect_simd_level()); +} + +static void dispatch_binary_right_scalar_for_level_f32(lw_scalar_binary_op operation, + const float* left, float right, + float* output, uint64_t element_count, + lw_simd_level simd_level) { + if (operation != LW_SCALAR_BINARY_ADD && operation != LW_SCALAR_BINARY_MUL && + operation != LW_SCALAR_BINARY_DIV) { + lw_scalar_binary_right_scalar_f32(operation, left, right, output, element_count); + } else if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_binary_right_scalar_f32(operation, left, right, output, element_count); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_binary_right_scalar_f32(operation, left, right, output, element_count); + } else { + lw_scalar_binary_right_scalar_f32(operation, left, right, output, element_count); + } +} + +static void dispatch_binary_right_scalar_f32(lw_scalar_binary_op operation, const float* left, + float right, float* output, uint64_t element_count) { + dispatch_binary_right_scalar_for_level_f32(operation, left, right, output, element_count, + lw_detect_simd_level()); +} + +static int try_dispatch_binary_single_axis_broadcast_f32( + lw_scalar_binary_op operation, const float* left, const float* right, float* output, + uint32_t right_rank, const int32_t* right_dimensions, uint32_t output_rank, + const int32_t* output_dimensions, uint64_t output_count) { + uint32_t right_padding = output_rank - right_rank; + uint32_t non_unit_dimensions = 0u; + uint32_t broadcast_axis = 0u; + uint32_t axis; + uint64_t inner_count = 1u; + uint64_t right_dimension; + uint64_t outer_count; + uint64_t outer_index; + lw_simd_level simd_level; + + /* Many OCR graph biases vary on exactly one axis. Recognizing that shape + * turns a general coordinate walk into contiguous vector operations. */ + for (axis = 0u; axis < output_rank; ++axis) { + int32_t dimension = axis < right_padding ? 1 : right_dimensions[axis - right_padding]; + if (dimension != 1) { + ++non_unit_dimensions; + broadcast_axis = axis; + } + } + if (non_unit_dimensions != 1u) { + return 0; + } + for (axis = broadcast_axis + 1u; axis < output_rank; ++axis) { + inner_count *= (uint32_t)output_dimensions[axis]; + } + right_dimension = (uint32_t)output_dimensions[broadcast_axis]; + outer_count = output_count / (right_dimension * inner_count); + simd_level = lw_detect_simd_level(); + for (outer_index = 0u; outer_index < outer_count; ++outer_index) { + uint64_t outer_offset = outer_index * right_dimension * inner_count; + if (inner_count == 1u) { + dispatch_binary_contiguous_for_level_f32(operation, left + (size_t)outer_offset, right, + output + (size_t)outer_offset, right_dimension, + simd_level); + } else { + uint64_t right_index; + for (right_index = 0u; right_index < right_dimension; ++right_index) { + uint64_t block_offset = outer_offset + right_index * inner_count; + dispatch_binary_right_scalar_for_level_f32( + operation, left + (size_t)block_offset, right[(size_t)right_index], + output + (size_t)block_offset, inner_count, simd_level); + } + } + } + return 1; +} + +lw_status lw_scalar_binary_f32(lw_scalar_binary_op operation, const float* left, uint32_t left_rank, + const int32_t* left_dimensions, const float* right, + uint32_t right_rank, const int32_t* right_dimensions, float* output, + uint32_t output_rank, const int32_t* output_dimensions) { + uint64_t left_contiguous[LW_MAX_DIMS] = {0u}; + uint64_t right_contiguous[LW_MAX_DIMS] = {0u}; + uint64_t left_strides[LW_MAX_DIMS] = {0u}; + uint64_t right_strides[LW_MAX_DIMS] = {0u}; + uint32_t coordinates[LW_MAX_DIMS] = {0u}; + uint64_t left_count; + uint64_t right_count; + uint64_t output_count = 1u; + uint64_t left_offset = 0u; + uint64_t right_offset = 0u; + uint64_t output_index; + uint32_t axis; + lw_status status; + + if (operation != LW_SCALAR_BINARY_ADD && operation != LW_SCALAR_BINARY_MUL && + operation != LW_SCALAR_BINARY_DIV && operation != LW_SCALAR_BINARY_SUB && + operation != LW_SCALAR_BINARY_POW) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (left == NULL || right == NULL || output == NULL || output == left || output == right) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (left_rank > LW_MAX_DIMS || right_rank > LW_MAX_DIMS || output_rank > LW_MAX_DIMS || + output_rank != (left_rank > right_rank ? left_rank : right_rank) || + (output_rank != 0u && output_dimensions == NULL)) { + return LW_STATUS_INVALID_SHAPE; + } + status = make_contiguous_strides(left_rank, left_dimensions, left_contiguous, &left_count); + if (status != LW_STATUS_OK) { + return status; + } + status = make_contiguous_strides(right_rank, right_dimensions, right_contiguous, &right_count); + if (status != LW_STATUS_OK) { + return status; + } + for (axis = 0u; axis < output_rank; ++axis) { + uint32_t left_padding = output_rank - left_rank; + uint32_t right_padding = output_rank - right_rank; + int32_t left_dimension = axis < left_padding ? 1 : left_dimensions[axis - left_padding]; + int32_t right_dimension = axis < right_padding ? 1 : right_dimensions[axis - right_padding]; + int32_t expected_dimension; + if (left_dimension <= 0 || right_dimension <= 0 || + (left_dimension != right_dimension && left_dimension != 1 && right_dimension != 1)) { + return LW_STATUS_INVALID_SHAPE; + } + expected_dimension = left_dimension > right_dimension ? left_dimension : right_dimension; + if (output_dimensions[axis] != expected_dimension) { + return LW_STATUS_INVALID_SHAPE; + } + if (output_count > UINT64_MAX / (uint32_t)expected_dimension) { + return LW_STATUS_OUT_OF_BOUNDS; + } + output_count *= (uint32_t)expected_dimension; + if (axis >= left_padding && left_dimension != 1) { + left_strides[axis] = left_contiguous[axis - left_padding]; + } + if (axis >= right_padding && right_dimension != 1) { + right_strides[axis] = right_contiguous[axis - right_padding]; + } + } + if (output_count > (uint64_t)(SIZE_MAX / sizeof(float))) { + return LW_STATUS_OUT_OF_BOUNDS; + } + + /* Prefer simple contiguous cases before the generic broadcasting loop. */ + if (left_count == output_count && right_count == output_count) { + dispatch_binary_contiguous_f32(operation, left, right, output, output_count); + return LW_STATUS_OK; + } + if (left_count == output_count && right_count == 1u) { + dispatch_binary_right_scalar_f32(operation, left, right[0], output, output_count); + return LW_STATUS_OK; + } + if (left_count == output_count && + try_dispatch_binary_single_axis_broadcast_f32(operation, left, right, output, right_rank, + right_dimensions, output_rank, + output_dimensions, output_count)) { + return LW_STATUS_OK; + } + + for (output_index = 0u; output_index < output_count; ++output_index) { + float left_value = left[(size_t)left_offset]; + float right_value = right[(size_t)right_offset]; + if (operation == LW_SCALAR_BINARY_ADD) { + output[(size_t)output_index] = left_value + right_value; + } else if (operation == LW_SCALAR_BINARY_MUL) { + output[(size_t)output_index] = left_value * right_value; + } else { + output[(size_t)output_index] = left_value / right_value; + } + + for (axis = output_rank; axis > 0u; --axis) { + uint32_t current_axis = axis - 1u; + ++coordinates[current_axis]; + if (coordinates[current_axis] < (uint32_t)output_dimensions[current_axis]) { + left_offset += left_strides[current_axis]; + right_offset += right_strides[current_axis]; + break; + } + coordinates[current_axis] = 0u; + left_offset -= + left_strides[current_axis] * ((uint32_t)output_dimensions[current_axis] - 1u); + right_offset -= + right_strides[current_axis] * ((uint32_t)output_dimensions[current_axis] - 1u); + } + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_kernels.h b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_kernels.h new file mode 100644 index 00000000..128dc64b --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_kernels.h @@ -0,0 +1,146 @@ +#ifndef LW_SCALAR_KERNELS_H +#define LW_SCALAR_KERNELS_H + +/* + * Internal numeric-kernel contract. The scalar entry points are the portable + * correctness baseline; selected wrappers may dispatch to SSE2 or AVX2 only + * after validation has proved that dimensions and buffers are safe. + */ + +#include "lw_infer.h" + +#include + +typedef enum lw_scalar_binary_op { + LW_SCALAR_BINARY_ADD = 1, + LW_SCALAR_BINARY_MUL = 2, + LW_SCALAR_BINARY_DIV = 3, + LW_SCALAR_BINARY_SUB = 4, + LW_SCALAR_BINARY_POW = 5 +} lw_scalar_binary_op; + +void lw_scalar_binary_contiguous_f32(lw_scalar_binary_op operation, const float* left, + const float* right, float* output, uint64_t element_count); +void lw_scalar_binary_right_scalar_f32(lw_scalar_binary_op operation, const float* left, + float right, float* output, uint64_t element_count); + +lw_status lw_scalar_binary_f32(lw_scalar_binary_op operation, const float* left, uint32_t left_rank, + const int32_t* left_dimensions, const float* right, + uint32_t right_rank, const int32_t* right_dimensions, float* output, + uint32_t output_rank, const int32_t* output_dimensions); + +lw_status lw_scalar_relu_f32(const float* input, float* output, uint64_t element_count); +lw_status lw_scalar_erf_f32(const float* input, float* output, uint64_t element_count); +lw_status lw_scalar_hard_sigmoid_f32(const float* input, float* output, uint64_t element_count, + float alpha, float beta); +lw_status lw_scalar_sigmoid_f32(const float* input, float* output, uint64_t element_count); +lw_status lw_scalar_sqrt_f32(const float* input, float* output, uint64_t element_count); +lw_status lw_scalar_softmax_f32(const float* input, float* output, uint32_t rank, + const int32_t* dimensions, int32_t axis); +lw_status lw_ctc_greedy_softmax_contiguous_f32(const float* input, uint32_t* best_indices, + float* emitted_probabilities, + uint64_t row_count, uint64_t axis_count); +lw_status lw_ctc_emitted_softmax_contiguous_f32(const float* input, + const uint32_t* best_indices, + float* emitted_probabilities, + uint64_t row_count, uint64_t axis_count); +lw_status lw_scalar_transpose_f32(const float* input, float* output, uint32_t rank, + const int32_t* input_dimensions, uint32_t permutation_count, + const int32_t* permutation, const int32_t* output_dimensions); +lw_status lw_scalar_squeeze_f32(const float* input, float* output, uint32_t input_rank, + const int32_t* input_dimensions, uint32_t axes_count, + const int32_t* axes, uint32_t output_rank, + const int32_t* output_dimensions); +lw_status lw_scalar_unsqueeze_f32(const float* input, float* output, uint32_t input_rank, + const int32_t* input_dimensions, uint32_t axes_count, + const int32_t* axes, uint32_t output_rank, + const int32_t* output_dimensions); +lw_status lw_scalar_reshape_f32(const float* input, float* output, uint32_t input_rank, + const int32_t* input_dimensions, uint32_t output_rank, + const int32_t* output_dimensions); +lw_status lw_scalar_concat_f32(const float* const* inputs, uint32_t input_count, + const uint32_t* input_ranks, const int32_t* const* input_dimensions, + float* output, uint32_t output_rank, + const int32_t* output_dimensions, int32_t axis); +lw_status lw_scalar_resize_nearest_f32(const float* input, float* output, uint32_t rank, + const int32_t* input_dimensions, + const int32_t* output_dimensions, const float* scales); +lw_status lw_scalar_slice_f32(const float* input, float* output, uint32_t rank, + const int32_t* input_dimensions, const int32_t* output_dimensions, + uint32_t slice_count, const int32_t* starts, const int32_t* ends, + const int32_t* axes, const int32_t* steps); +lw_status lw_scalar_reduce_mean_f32(const float* input, float* output, uint32_t input_rank, + const int32_t* input_dimensions, uint32_t axes_count, + const int32_t* axes, uint32_t keep_dimensions, + uint32_t no_op_with_empty_axes, uint32_t output_rank, + const int32_t* output_dimensions); +lw_status lw_scalar_average_pool2d_f32(const float* input, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4], const int32_t kernel[2], + const int32_t strides[2], const int32_t pads[4], + uint32_t ceil_mode, uint32_t count_include_pad); +lw_status lw_scalar_max_pool2d_f32(const float* input, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4], const int32_t kernel[2], + const int32_t strides[2], const int32_t pads[4], + uint32_t ceil_mode); +lw_status lw_scalar_matmul_shared_f32(const float* input, const float* weights, float* output, + uint32_t batch_count, uint32_t rows, uint32_t inner_dimension, + uint32_t columns); +lw_status lw_scalar_matmul_f32(const float* input, const float* weights, float* output, + uint32_t input_rank, const int32_t* input_dimensions, + uint32_t weights_rank, const int32_t* weights_dimensions, + uint32_t output_rank, const int32_t* output_dimensions); +lw_status lw_matmul_shared_f32(const float* input, const float* weights, float* output, + uint32_t batch_count, uint32_t rows, uint32_t inner_dimension, + uint32_t columns); +lw_status lw_scalar_conv2d_f32(const float* input, const float* weights, const float* bias, + uint32_t bias_count, float* output, + const int32_t input_dimensions[4], + const int32_t weight_dimensions[4], + const int32_t output_dimensions[4], const int32_t kernel[2], + const int32_t strides[2], const int32_t dilations[2], + const int32_t pads[4], uint32_t groups); +lw_status lw_scalar_conv_transpose2d_f32( + const float* input, const float* weights, const float* bias, uint32_t bias_count, float* output, + const int32_t input_dimensions[4], const int32_t weight_dimensions[4], + const int32_t output_dimensions[4], const int32_t kernel[2], const int32_t strides[2], + const int32_t dilations[2], const int32_t pads[4], uint32_t groups); +void lw_scalar_conv_transpose2x2_stride2_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); +void lw_scalar_conv_transpose2x2_stride2_range_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4], + uint32_t output_channel_begin, uint32_t output_channel_end); +void lw_scalar_conv1x1_unit_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4], uint32_t groups, + uint32_t input_channels_per_group, + uint32_t output_channels_per_group); +void lw_scalar_depthwise_conv3x3_unit_pad1_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]); +void lw_scalar_depthwise_conv3x3_stride2x1_pad1_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); +void lw_scalar_depthwise_conv5x5_unit_pad2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]); +void lw_scalar_conv3x3_unit_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_scalar_conv2x2_unit_pad_end1_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_scalar_conv3x3_stride2_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +lw_status lw_scalar_batch_normalization_f32(const float* input, const float* scale, + const float* bias, const float* mean, + const float* variance, uint32_t parameter_count, + float epsilon, float* output, uint32_t rank, + const int32_t* dimensions); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_layout.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_layout.c new file mode 100644 index 00000000..d738cbf1 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_layout.c @@ -0,0 +1,506 @@ +#include "scalar_kernels.h" + +/* Shape-preserving data movement: transpose, reshape, concat and resize. */ + +#include +#include +#include +#include + +static int normalize_axis(int32_t axis, uint32_t rank, uint32_t* normalized) { + int64_t value = axis; + if (value < 0) { + value += rank; + } + if (value < 0 || value >= rank) { + return 0; + } + *normalized = (uint32_t)value; + return 1; +} + +static lw_status tensor_element_count(uint32_t rank, const int32_t* dimensions, uint64_t* count) { + uint64_t value = 1u; + uint32_t index; + if (rank > LW_MAX_DIMS || (rank != 0u && dimensions == NULL)) { + return LW_STATUS_INVALID_SHAPE; + } + for (index = 0u; index < rank; ++index) { + if (dimensions[index] <= 0 || value > UINT64_MAX / (uint32_t)dimensions[index]) { + return LW_STATUS_INVALID_SHAPE; + } + value *= (uint32_t)dimensions[index]; + } + if (value > (uint64_t)(SIZE_MAX / sizeof(float))) { + return LW_STATUS_OUT_OF_BOUNDS; + } + *count = value; + return LW_STATUS_OK; +} + +static lw_status copy_reshape(const float* input, float* output, uint64_t element_count) { + if (input == NULL || output == NULL) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (input != output) { + memcpy(output, input, (size_t)element_count * sizeof(float)); + } + return LW_STATUS_OK; +} + +static int integer_resize_factor(float scale, int32_t input_dimension, int32_t output_dimension, + uint32_t* factor) { + uint32_t candidate; + if (output_dimension % input_dimension != 0) { + return 0; + } + candidate = (uint32_t)(output_dimension / input_dimension); + if (candidate == 0u || (float)candidate != scale) { + return 0; + } + *factor = candidate; + return 1; +} + +static void resize_nchw_integer_spatial_f32(const float* input, float* output, + const int32_t* input_dimensions, + const int32_t* output_dimensions, + uint32_t height_factor, uint32_t width_factor) { + const uint32_t batch_count = (uint32_t)input_dimensions[0]; + const uint32_t channels = (uint32_t)input_dimensions[1]; + const uint32_t input_height = (uint32_t)input_dimensions[2]; + const uint32_t input_width = (uint32_t)input_dimensions[3]; + const uint32_t output_height = (uint32_t)output_dimensions[2]; + const uint32_t output_width = (uint32_t)output_dimensions[3]; + uint32_t batch; + for (batch = 0u; batch < batch_count; ++batch) { + uint32_t channel; + for (channel = 0u; channel < channels; ++channel) { + uint64_t input_plane = + ((uint64_t)batch * channels + channel) * input_height * input_width; + uint64_t output_plane = + ((uint64_t)batch * channels + channel) * output_height * output_width; + uint32_t input_y; + for (input_y = 0u; input_y < input_height; ++input_y) { + const float* source_row = input + (size_t)(input_plane + input_y * input_width); + float* first_output_row = + output + + (size_t)(output_plane + (uint64_t)input_y * height_factor * output_width); + uint32_t input_x; + for (input_x = 0u; input_x < input_width; ++input_x) { + uint32_t repeat_x; + for (repeat_x = 0u; repeat_x < width_factor; ++repeat_x) { + first_output_row[(size_t)input_x * width_factor + repeat_x] = + source_row[input_x]; + } + } + for (uint32_t repeat_y = 1u; repeat_y < height_factor; ++repeat_y) { + memcpy(first_output_row + (size_t)repeat_y * output_width, first_output_row, + (size_t)output_width * sizeof(float)); + } + } + } + } +} + +lw_status lw_scalar_transpose_f32(const float* input, float* output, uint32_t rank, + const int32_t* input_dimensions, uint32_t permutation_count, + const int32_t* permutation, const int32_t* output_dimensions) { + uint64_t input_strides[LW_MAX_DIMS] = {0u}; + int used[LW_MAX_DIMS] = {0}; + uint64_t element_count; + uint64_t stride = 1u; + uint64_t output_index; + uint32_t axis; + lw_status status; + if (input == NULL || output == NULL || input == output) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (rank > LW_MAX_DIMS || permutation_count != rank || + (rank != 0u && (permutation == NULL || output_dimensions == NULL))) { + return LW_STATUS_INVALID_SHAPE; + } + status = tensor_element_count(rank, input_dimensions, &element_count); + if (status != LW_STATUS_OK) { + return status; + } + for (axis = rank; axis > 0u; --axis) { + input_strides[axis - 1u] = stride; + stride *= (uint32_t)input_dimensions[axis - 1u]; + } + for (axis = 0u; axis < rank; ++axis) { + int32_t source_axis = permutation[axis]; + if (source_axis < 0 || (uint32_t)source_axis >= rank || used[source_axis] || + output_dimensions[axis] != input_dimensions[source_axis]) { + return LW_STATUS_INVALID_SHAPE; + } + used[source_axis] = 1; + } + for (output_index = 0u; output_index < element_count; ++output_index) { + uint64_t remaining = output_index; + uint64_t input_offset = 0u; + for (axis = rank; axis > 0u; --axis) { + uint32_t output_axis = axis - 1u; + uint32_t coordinate = (uint32_t)(remaining % (uint32_t)output_dimensions[output_axis]); + remaining /= (uint32_t)output_dimensions[output_axis]; + input_offset += + (uint64_t)coordinate * input_strides[(uint32_t)permutation[output_axis]]; + } + output[(size_t)output_index] = input[(size_t)input_offset]; + } + return LW_STATUS_OK; +} + +lw_status lw_scalar_squeeze_f32(const float* input, float* output, uint32_t input_rank, + const int32_t* input_dimensions, uint32_t axes_count, + const int32_t* axes, uint32_t output_rank, + const int32_t* output_dimensions) { + int squeezed[LW_MAX_DIMS] = {0}; + uint64_t input_count; + uint64_t output_count; + uint32_t axis; + uint32_t output_axis = 0u; + lw_status status; + if (axes_count > LW_MAX_DIMS || (axes_count != 0u && axes == NULL)) { + return LW_STATUS_INVALID_SHAPE; + } + status = tensor_element_count(input_rank, input_dimensions, &input_count); + if (status != LW_STATUS_OK) { + return status; + } + if (axes_count == 0u) { + for (axis = 0u; axis < input_rank; ++axis) { + squeezed[axis] = input_dimensions[axis] == 1; + } + } else { + for (axis = 0u; axis < axes_count; ++axis) { + uint32_t normalized; + if (!normalize_axis(axes[axis], input_rank, &normalized) || squeezed[normalized] || + input_dimensions[normalized] != 1) { + return LW_STATUS_INVALID_SHAPE; + } + squeezed[normalized] = 1; + } + } + for (axis = 0u; axis < input_rank; ++axis) { + if (!squeezed[axis]) { + if (output_axis >= output_rank || output_dimensions == NULL || + output_dimensions[output_axis] != input_dimensions[axis]) { + return LW_STATUS_INVALID_SHAPE; + } + ++output_axis; + } + } + if (output_axis != output_rank) { + return LW_STATUS_INVALID_SHAPE; + } + status = tensor_element_count(output_rank, output_dimensions, &output_count); + if (status != LW_STATUS_OK || output_count != input_count) { + return status == LW_STATUS_OK ? LW_STATUS_INVALID_SHAPE : status; + } + return copy_reshape(input, output, input_count); +} + +lw_status lw_scalar_unsqueeze_f32(const float* input, float* output, uint32_t input_rank, + const int32_t* input_dimensions, uint32_t axes_count, + const int32_t* axes, uint32_t output_rank, + const int32_t* output_dimensions) { + int inserted[LW_MAX_DIMS] = {0}; + uint64_t input_count; + uint64_t output_count; + uint32_t axis; + uint32_t input_axis = 0u; + lw_status status; + if (input_rank > LW_MAX_DIMS || axes_count > LW_MAX_DIMS || + input_rank + axes_count != output_rank || output_rank > LW_MAX_DIMS || + (axes_count != 0u && axes == NULL)) { + return LW_STATUS_INVALID_SHAPE; + } + status = tensor_element_count(input_rank, input_dimensions, &input_count); + if (status != LW_STATUS_OK) { + return status; + } + for (axis = 0u; axis < axes_count; ++axis) { + uint32_t normalized; + if (!normalize_axis(axes[axis], output_rank, &normalized) || inserted[normalized]) { + return LW_STATUS_INVALID_SHAPE; + } + inserted[normalized] = 1; + } + for (axis = 0u; axis < output_rank; ++axis) { + int32_t expected = inserted[axis] ? 1 : input_dimensions[input_axis++]; + if (output_dimensions == NULL || output_dimensions[axis] != expected) { + return LW_STATUS_INVALID_SHAPE; + } + } + status = tensor_element_count(output_rank, output_dimensions, &output_count); + if (status != LW_STATUS_OK || output_count != input_count) { + return status == LW_STATUS_OK ? LW_STATUS_INVALID_SHAPE : status; + } + return copy_reshape(input, output, input_count); +} + +lw_status lw_scalar_reshape_f32(const float* input, float* output, uint32_t input_rank, + const int32_t* input_dimensions, uint32_t output_rank, + const int32_t* output_dimensions) { + uint64_t input_count; + uint64_t output_count; + lw_status status = tensor_element_count(input_rank, input_dimensions, &input_count); + if (status != LW_STATUS_OK) { + return status; + } + status = tensor_element_count(output_rank, output_dimensions, &output_count); + if (status != LW_STATUS_OK) { + return status; + } + if (input_count != output_count) { + return LW_STATUS_INVALID_SHAPE; + } + return copy_reshape(input, output, input_count); +} + +lw_status lw_scalar_concat_f32(const float* const* inputs, uint32_t input_count, + const uint32_t* input_ranks, const int32_t* const* input_dimensions, + float* output, uint32_t output_rank, + const int32_t* output_dimensions, int32_t axis) { + uint32_t normalized_axis; + uint64_t outer = 1u; + uint64_t inner = 1u; + uint64_t output_count; + uint64_t concatenated_axis = 0u; + uint32_t input_index; + uint32_t dimension_index; + uint64_t outer_index; + lw_status status; + if (inputs == NULL || input_ranks == NULL || input_dimensions == NULL || output == NULL || + output_dimensions == NULL || input_count < 1u || + !normalize_axis(axis, output_rank, &normalized_axis)) { + return LW_STATUS_INVALID_ARGUMENT; + } + status = tensor_element_count(output_rank, output_dimensions, &output_count); + if (status != LW_STATUS_OK) { + return status; + } + (void)output_count; + for (dimension_index = 0u; dimension_index < normalized_axis; ++dimension_index) { + outer *= (uint32_t)output_dimensions[dimension_index]; + } + for (dimension_index = normalized_axis + 1u; dimension_index < output_rank; ++dimension_index) { + inner *= (uint32_t)output_dimensions[dimension_index]; + } + for (input_index = 0u; input_index < input_count; ++input_index) { + uint64_t ignored_count; + if (inputs[input_index] == NULL || inputs[input_index] == output || + input_ranks[input_index] != output_rank || input_dimensions[input_index] == NULL) { + return LW_STATUS_INVALID_ARGUMENT; + } + status = tensor_element_count(input_ranks[input_index], input_dimensions[input_index], + &ignored_count); + if (status != LW_STATUS_OK) { + return status; + } + for (dimension_index = 0u; dimension_index < output_rank; ++dimension_index) { + if (dimension_index != normalized_axis && + input_dimensions[input_index][dimension_index] != + output_dimensions[dimension_index]) { + return LW_STATUS_INVALID_SHAPE; + } + } + concatenated_axis += (uint32_t)input_dimensions[input_index][normalized_axis]; + } + if (concatenated_axis != (uint32_t)output_dimensions[normalized_axis]) { + return LW_STATUS_INVALID_SHAPE; + } + for (outer_index = 0u; outer_index < outer; ++outer_index) { + uint64_t output_axis_offset = 0u; + for (input_index = 0u; input_index < input_count; ++input_index) { + uint64_t axis_elements = + (uint64_t)(uint32_t)input_dimensions[input_index][normalized_axis] * inner; + uint64_t input_offset = outer_index * axis_elements; + uint64_t output_offset = + (outer_index * (uint32_t)output_dimensions[normalized_axis] * inner) + + output_axis_offset; + memcpy(output + (size_t)output_offset, inputs[input_index] + (size_t)input_offset, + (size_t)axis_elements * sizeof(float)); + output_axis_offset += axis_elements; + } + } + return LW_STATUS_OK; +} + +lw_status lw_scalar_resize_nearest_f32(const float* input, float* output, uint32_t rank, + const int32_t* input_dimensions, + const int32_t* output_dimensions, const float* scales) { + uint64_t input_count; + uint64_t output_count; + uint64_t output_index; + uint64_t input_strides[LW_MAX_DIMS] = {0u}; + uint64_t stride = 1u; + uint32_t height_factor; + uint32_t width_factor; + uint32_t axis; + lw_status status; + if (input == NULL || output == NULL || input == output || scales == NULL || rank == 0u || + rank > LW_MAX_DIMS || input_dimensions == NULL || output_dimensions == NULL) { + return LW_STATUS_INVALID_ARGUMENT; + } + status = tensor_element_count(rank, input_dimensions, &input_count); + if (status != LW_STATUS_OK) { + return status; + } + status = tensor_element_count(rank, output_dimensions, &output_count); + if (status != LW_STATUS_OK) { + return status; + } + (void)input_count; + for (axis = rank; axis > 0u; --axis) { + input_strides[axis - 1u] = stride; + stride *= (uint32_t)input_dimensions[axis - 1u]; + } + for (axis = 0u; axis < rank; ++axis) { + double expected; + if (!isfinite(scales[axis]) || scales[axis] <= 0.0f) { + return LW_STATUS_INVALID_SHAPE; + } + expected = floor((double)input_dimensions[axis] * scales[axis]); + if (expected != output_dimensions[axis]) { + return LW_STATUS_INVALID_SHAPE; + } + } + if (rank == 4u && input_dimensions[0] == output_dimensions[0] && + input_dimensions[1] == output_dimensions[1] && scales[0] == 1.0f && scales[1] == 1.0f && + integer_resize_factor(scales[2], input_dimensions[2], output_dimensions[2], + &height_factor) && + integer_resize_factor(scales[3], input_dimensions[3], output_dimensions[3], + &width_factor)) { + resize_nchw_integer_spatial_f32(input, output, input_dimensions, output_dimensions, + height_factor, width_factor); + return LW_STATUS_OK; + } + for (output_index = 0u; output_index < output_count; ++output_index) { + uint64_t remaining = output_index; + uint64_t input_offset = 0u; + for (axis = rank; axis > 0u; --axis) { + uint32_t current_axis = axis - 1u; + uint32_t coordinate = (uint32_t)(remaining % (uint32_t)output_dimensions[current_axis]); + uint32_t input_coordinate = (uint32_t)floor((double)coordinate / scales[current_axis]); + if (input_coordinate >= (uint32_t)input_dimensions[current_axis]) { + input_coordinate = (uint32_t)input_dimensions[current_axis] - 1u; + } + remaining /= (uint32_t)output_dimensions[current_axis]; + input_offset += (uint64_t)input_coordinate * input_strides[current_axis]; + } + output[(size_t)output_index] = input[(size_t)input_offset]; + } + return LW_STATUS_OK; +} + +static int slice_bounds(int32_t dimension, int32_t start, int32_t end, int32_t step, + int32_t* first, int32_t* length) { + int64_t normalized_start = start; + int64_t normalized_end = end; + int64_t count; + if (dimension <= 0 || step <= 0) { + return 0; + } + if (normalized_start == INT32_MIN) { + normalized_start = 0; + } else if (normalized_start < 0) { + normalized_start += dimension; + } + if (normalized_end == INT32_MAX) { + normalized_end = dimension; + } else if (normalized_end < 0) { + normalized_end += dimension; + } + if (normalized_start < 0) { + normalized_start = 0; + } else if (normalized_start > dimension) { + normalized_start = dimension; + } + if (normalized_end < 0) { + normalized_end = 0; + } else if (normalized_end > dimension) { + normalized_end = dimension; + } + count = normalized_end <= normalized_start + ? 0 + : (normalized_end - normalized_start + step - 1) / step; + if (count <= 0 || count > INT32_MAX) { + return 0; + } + *first = (int32_t)normalized_start; + *length = (int32_t)count; + return 1; +} + +lw_status lw_scalar_slice_f32(const float* input, float* output, uint32_t rank, + const int32_t* input_dimensions, const int32_t* output_dimensions, + uint32_t slice_count, const int32_t* starts, const int32_t* ends, + const int32_t* axes, const int32_t* steps) { + uint64_t input_count; + uint64_t output_count; + uint64_t input_strides[LW_MAX_DIMS] = {0u}; + int32_t first[LW_MAX_DIMS]; + int32_t lengths[LW_MAX_DIMS]; + int32_t slice_steps[LW_MAX_DIMS]; + int selected[LW_MAX_DIMS] = {0}; + uint64_t stride = 1u; + uint64_t output_index; + uint32_t axis; + lw_status status; + if (input == NULL || output == NULL || input == output || input_dimensions == NULL || + output_dimensions == NULL || starts == NULL || ends == NULL || axes == NULL || + steps == NULL || rank == 0u || rank > LW_MAX_DIMS || slice_count == 0u || + slice_count > rank) { + return LW_STATUS_INVALID_ARGUMENT; + } + status = tensor_element_count(rank, input_dimensions, &input_count); + if (status != LW_STATUS_OK) { + return status; + } + status = tensor_element_count(rank, output_dimensions, &output_count); + if (status != LW_STATUS_OK) { + return status; + } + for (axis = rank; axis > 0u; --axis) { + input_strides[axis - 1u] = stride; + stride *= (uint32_t)input_dimensions[axis - 1u]; + } + for (axis = 0u; axis < rank; ++axis) { + first[axis] = 0; + lengths[axis] = input_dimensions[axis]; + slice_steps[axis] = 1; + } + for (axis = 0u; axis < slice_count; ++axis) { + uint32_t normalized_axis; + if (!normalize_axis(axes[axis], rank, &normalized_axis) || selected[normalized_axis] || + !slice_bounds(input_dimensions[normalized_axis], starts[axis], ends[axis], + steps[axis], &first[normalized_axis], &lengths[normalized_axis])) { + return LW_STATUS_INVALID_SHAPE; + } + selected[normalized_axis] = 1; + slice_steps[normalized_axis] = steps[axis]; + } + for (axis = 0u; axis < rank; ++axis) { + if (output_dimensions[axis] != lengths[axis]) { + return LW_STATUS_INVALID_SHAPE; + } + } + (void)input_count; + for (output_index = 0u; output_index < output_count; ++output_index) { + uint64_t remaining = output_index; + uint64_t input_offset = 0u; + for (axis = rank; axis > 0u; --axis) { + uint32_t current_axis = axis - 1u; + uint32_t coordinate = + (uint32_t)(remaining % (uint32_t)output_dimensions[current_axis]); + remaining /= (uint32_t)output_dimensions[current_axis]; + input_offset += (uint64_t)(first[current_axis] + + coordinate * (uint32_t)slice_steps[current_axis]) * + input_strides[current_axis]; + } + output[(size_t)output_index] = input[(size_t)input_offset]; + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_matmul.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_matmul.c new file mode 100644 index 00000000..73f03ccc --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_matmul.c @@ -0,0 +1,253 @@ +#include "scalar_kernels.h" + +/* Shared-weight matrix multiplication used by the converted OCR graphs. */ +#include "cpu_features.h" +#include "simd_kernels.h" + +#include +#include + +static int multiply_fits(uint64_t left, uint64_t right, uint64_t* result) { + if (right != 0u && left > UINT64_MAX / right) { + return 0; + } + *result = left * right; + return 1; +} + +static int buffer_fits(uint32_t first, uint32_t second, uint32_t third) { + uint64_t count; + if (!multiply_fits(first, second, &count) || !multiply_fits(count, third, &count)) { + return 0; + } + return count <= (uint64_t)(SIZE_MAX / sizeof(float)); +} + +static lw_status validate_matmul_shared_f32(const float* input, const float* weights, float* output, + uint32_t batch_count, uint32_t rows, + uint32_t inner_dimension, uint32_t columns) { + if (input == NULL || weights == NULL || output == NULL || output == input || + output == weights) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (batch_count == 0u || rows == 0u || inner_dimension == 0u || columns == 0u) { + return LW_STATUS_INVALID_SHAPE; + } + if (!buffer_fits(batch_count, rows, inner_dimension) || + !buffer_fits(1u, inner_dimension, columns) || !buffer_fits(batch_count, rows, columns)) { + return LW_STATUS_OUT_OF_BOUNDS; + } + return LW_STATUS_OK; +} + +static void scalar_matmul_shared_f32(const float* input, const float* weights, float* output, + uint32_t batch_count, uint32_t rows, uint32_t inner_dimension, + uint32_t columns) { + uint32_t batch; + uint32_t row; + uint32_t column; + /* Four output rows share each sequential scan of the weight matrix. This + * improves cache locality while remaining the scalar correctness path. */ + for (batch = 0u; batch < batch_count; ++batch) { + for (row = 0u; row < rows;) { + uint32_t row_end = rows - row < 4u ? rows : row + 4u; + uint32_t inner; + uint32_t current_row; + for (current_row = row; current_row < row_end; ++current_row) { + uint64_t output_base = ((uint64_t)batch * rows + current_row) * columns; + for (column = 0u; column < columns; ++column) { + output[(size_t)(output_base + column)] = 0.0f; + } + } + for (inner = 0u; inner < inner_dimension; ++inner) { + uint64_t weight_base = (uint64_t)inner * columns; + for (current_row = row; current_row < row_end; ++current_row) { + uint64_t input_base = ((uint64_t)batch * rows + current_row) * inner_dimension; + uint64_t output_base = ((uint64_t)batch * rows + current_row) * columns; + float input_value = input[(size_t)(input_base + inner)]; + for (column = 0u; column < columns; ++column) { + output[(size_t)(output_base + column)] += + input_value * weights[(size_t)(weight_base + column)]; + } + } + } + row = row_end; + } + } +} + +static int matmul_shape_product(const int32_t* dimensions, uint32_t rank, uint64_t* result) { + uint32_t index; + uint64_t product = 1u; + if (dimensions == NULL || rank == 0u || rank > LW_MAX_DIMS) { + return 0; + } + for (index = 0u; index < rank; ++index) { + if (dimensions[index] <= 0 || product > UINT64_MAX / (uint32_t)dimensions[index]) { + return 0; + } + product *= (uint32_t)dimensions[index]; + } + *result = product; + return product <= (uint64_t)(SIZE_MAX / sizeof(float)); +} + +/* Portable ONNX MatMul for rank >= 2 tensors with broadcast batch dimensions. + * The common rank-N x rank-2 constant-weight case stays on the optimized + * shared-weight path in the executor; this fallback covers attention-style + * batched matrices used by larger REC graphs. */ +lw_status lw_scalar_matmul_f32(const float* input, const float* weights, float* output, + uint32_t input_rank, const int32_t* input_dimensions, + uint32_t weights_rank, const int32_t* weights_dimensions, + uint32_t output_rank, const int32_t* output_dimensions) { + uint32_t batch_rank; + uint32_t input_batch_rank; + uint32_t weights_batch_rank; + uint32_t input_offset; + uint32_t weights_offset; + uint32_t batch_index; + uint32_t batch_count; + uint32_t row; + uint32_t column; + uint32_t inner; + uint64_t input_elements; + uint64_t weights_elements; + uint64_t output_elements; + uint64_t input_matrix_size; + uint64_t weights_matrix_size; + uint64_t output_matrix_size; + if (input == NULL || weights == NULL || output == NULL || output == input || + output == weights || input_dimensions == NULL || weights_dimensions == NULL || + output_dimensions == NULL || input_rank < 2u || weights_rank < 2u || + input_rank > LW_MAX_DIMS || weights_rank > LW_MAX_DIMS || output_rank < 2u || + output_rank > LW_MAX_DIMS) { + return LW_STATUS_INVALID_ARGUMENT; + } + input_batch_rank = input_rank - 2u; + weights_batch_rank = weights_rank - 2u; + batch_rank = input_batch_rank > weights_batch_rank ? input_batch_rank : weights_batch_rank; + if (batch_rank > LW_MAX_DIMS - 2u || output_rank != batch_rank + 2u || + input_dimensions[input_rank - 1u] != weights_dimensions[weights_rank - 2u] || + input_dimensions[input_rank - 2u] != output_dimensions[batch_rank] || + weights_dimensions[weights_rank - 1u] != output_dimensions[batch_rank + 1u]) { + return LW_STATUS_INVALID_SHAPE; + } + input_offset = batch_rank - input_batch_rank; + weights_offset = batch_rank - weights_batch_rank; + batch_count = 1u; + for (batch_index = 0u; batch_index < batch_rank; ++batch_index) { + int32_t input_dimension = batch_index < input_offset + ? 1 + : input_dimensions[batch_index - input_offset]; + int32_t weights_dimension = batch_index < weights_offset + ? 1 + : weights_dimensions[batch_index - weights_offset]; + if (input_dimension <= 0 || weights_dimension <= 0 || + (input_dimension != weights_dimension && input_dimension != 1 && + weights_dimension != 1) || output_dimensions[batch_index] <= 0 || + (output_dimensions[batch_index] != input_dimension && + output_dimensions[batch_index] != weights_dimension)) { + return LW_STATUS_INVALID_SHAPE; + } + if (batch_count > UINT32_MAX / (uint32_t)output_dimensions[batch_index]) { + return LW_STATUS_OUT_OF_BOUNDS; + } + batch_count *= (uint32_t)output_dimensions[batch_index]; + } + if (!matmul_shape_product(input_dimensions, input_rank, &input_elements) || + !matmul_shape_product(weights_dimensions, weights_rank, &weights_elements) || + !matmul_shape_product(output_dimensions, output_rank, &output_elements)) { + return LW_STATUS_OUT_OF_BOUNDS; + } + (void)input_elements; + (void)weights_elements; + (void)output_elements; + input_matrix_size = (uint64_t)(uint32_t)input_dimensions[input_rank - 2u] * + (uint32_t)input_dimensions[input_rank - 1u]; + weights_matrix_size = (uint64_t)(uint32_t)weights_dimensions[weights_rank - 2u] * + (uint32_t)weights_dimensions[weights_rank - 1u]; + output_matrix_size = (uint64_t)(uint32_t)output_dimensions[batch_rank] * + (uint32_t)output_dimensions[batch_rank + 1u]; + for (batch_index = 0u; batch_index < batch_count; ++batch_index) { + uint32_t remaining = batch_index; + uint64_t input_batch = 0u; + uint64_t weights_batch = 0u; + uint32_t axis; + for (axis = batch_rank; axis-- > 0u;) { + uint32_t coordinate = remaining % (uint32_t)output_dimensions[axis]; + int32_t input_dimension = axis < input_offset + ? 1 + : input_dimensions[axis - input_offset]; + int32_t weights_dimension = axis < weights_offset + ? 1 + : weights_dimensions[axis - weights_offset]; + remaining /= (uint32_t)output_dimensions[axis]; + if (axis >= input_offset) { + input_batch = input_batch * (uint32_t)input_dimension + + (input_dimension == 1 ? 0u : coordinate); + } + if (axis >= weights_offset) { + weights_batch = weights_batch * (uint32_t)weights_dimension + + (weights_dimension == 1 ? 0u : coordinate); + } + } + for (row = 0u; row < (uint32_t)input_dimensions[input_rank - 2u]; ++row) { + for (column = 0u; column < (uint32_t)weights_dimensions[weights_rank - 1u]; + ++column) { + float value = 0.0f; + for (inner = 0u; inner < (uint32_t)input_dimensions[input_rank - 1u]; ++inner) { + size_t input_index = (size_t)(input_batch * input_matrix_size + + (uint64_t)row * + (uint32_t)input_dimensions[input_rank - 1u] + + inner); + size_t weights_index = (size_t)(weights_batch * weights_matrix_size + + (uint64_t)inner * + (uint32_t)weights_dimensions[weights_rank - 1u] + + column); + value += input[input_index] * weights[weights_index]; + } + output[(size_t)(batch_index * output_matrix_size + + (uint64_t)row * + (uint32_t)weights_dimensions[weights_rank - 1u] + + column)] = value; + } + } + } + return LW_STATUS_OK; +} + +lw_status lw_scalar_matmul_shared_f32(const float* input, const float* weights, float* output, + uint32_t batch_count, uint32_t rows, uint32_t inner_dimension, + uint32_t columns) { + lw_status status = validate_matmul_shared_f32(input, weights, output, batch_count, rows, + inner_dimension, columns); + if (status != LW_STATUS_OK) { + return status; + } + scalar_matmul_shared_f32(input, weights, output, batch_count, rows, inner_dimension, columns); + return LW_STATUS_OK; +} + +lw_status lw_matmul_shared_f32(const float* input, const float* weights, float* output, + uint32_t batch_count, uint32_t rows, uint32_t inner_dimension, + uint32_t columns) { + lw_simd_level simd_level; + lw_status status = validate_matmul_shared_f32(input, weights, output, batch_count, rows, + inner_dimension, columns); + if (status != LW_STATUS_OK) { + return status; + } + /* Detection happens at runtime so one binary remains safe on older CPUs. */ + simd_level = lw_detect_simd_level(); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_matmul_shared_f32(input, weights, output, batch_count, rows, inner_dimension, + columns); + } else if (lw_simd_level_is_sse2(simd_level)) { + lw_sse2_matmul_shared_f32(input, weights, output, batch_count, rows, inner_dimension, + columns); + } else { + scalar_matmul_shared_f32(input, weights, output, batch_count, rows, inner_dimension, + columns); + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_reduction.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_reduction.c new file mode 100644 index 00000000..ffcc6cdc --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_reduction.c @@ -0,0 +1,424 @@ +#include "scalar_kernels.h" + +/* Reduction and pooling kernels with explicit shape and bounds validation. */ + +#include +#include +#include +#include + +static int normalize_axis(int32_t axis, uint32_t rank, uint32_t* normalized) { + int64_t value = axis; + if (value < 0) { + value += rank; + } + if (value < 0 || value >= rank) { + return 0; + } + *normalized = (uint32_t)value; + return 1; +} + +static lw_status tensor_element_count(uint32_t rank, const int32_t* dimensions, uint64_t* count) { + uint64_t value = 1u; + uint32_t index; + if (rank > LW_MAX_DIMS || (rank != 0u && dimensions == NULL)) { + return LW_STATUS_INVALID_SHAPE; + } + for (index = 0u; index < rank; ++index) { + if (dimensions[index] <= 0 || value > UINT64_MAX / (uint32_t)dimensions[index]) { + return LW_STATUS_INVALID_SHAPE; + } + value *= (uint32_t)dimensions[index]; + } + if (value > (uint64_t)(SIZE_MAX / sizeof(float))) { + return LW_STATUS_OUT_OF_BOUNDS; + } + *count = value; + return LW_STATUS_OK; +} + +lw_status lw_scalar_reduce_mean_f32(const float* input, float* output, uint32_t input_rank, + const int32_t* input_dimensions, uint32_t axes_count, + const int32_t* axes, uint32_t keep_dimensions, + uint32_t no_op_with_empty_axes, uint32_t output_rank, + const int32_t* output_dimensions) { + int reduced[LW_MAX_DIMS] = {0}; + uint32_t input_to_output[LW_MAX_DIMS]; + uint32_t coordinates[LW_MAX_DIMS] = {0u}; + uint64_t output_strides[LW_MAX_DIMS] = {0u}; + uint64_t input_count; + uint64_t output_count; + uint64_t reduction_count = 1u; + uint64_t input_index; + uint64_t output_offset = 0u; + uint64_t stride = 1u; + uint32_t axis; + uint32_t expected_output_rank = 0u; + lw_status status; + if (input == NULL || output == NULL || input == output) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (axes_count > LW_MAX_DIMS || (axes_count != 0u && axes == NULL) || keep_dimensions > 1u || + no_op_with_empty_axes > 1u) { + return LW_STATUS_INVALID_SHAPE; + } + status = tensor_element_count(input_rank, input_dimensions, &input_count); + if (status != LW_STATUS_OK) { + return status; + } + if (axes_count == 0u && no_op_with_empty_axes == 0u) { + for (axis = 0u; axis < input_rank; ++axis) { + reduced[axis] = 1; + } + } else { + for (axis = 0u; axis < axes_count; ++axis) { + uint32_t normalized; + if (!normalize_axis(axes[axis], input_rank, &normalized) || reduced[normalized]) { + return LW_STATUS_INVALID_SHAPE; + } + reduced[normalized] = 1; + } + } + for (axis = 0u; axis < input_rank; ++axis) { + input_to_output[axis] = UINT32_MAX; + if (reduced[axis]) { + if (reduction_count > UINT64_MAX / (uint32_t)input_dimensions[axis]) { + return LW_STATUS_OUT_OF_BOUNDS; + } + reduction_count *= (uint32_t)input_dimensions[axis]; + if (keep_dimensions != 0u) { + if (expected_output_rank >= output_rank || output_dimensions == NULL || + output_dimensions[expected_output_rank] != 1) { + return LW_STATUS_INVALID_SHAPE; + } + ++expected_output_rank; + } + } else { + if (expected_output_rank >= output_rank || output_dimensions == NULL || + output_dimensions[expected_output_rank] != input_dimensions[axis]) { + return LW_STATUS_INVALID_SHAPE; + } + input_to_output[axis] = expected_output_rank++; + } + } + if (expected_output_rank != output_rank) { + return LW_STATUS_INVALID_SHAPE; + } + status = tensor_element_count(output_rank, output_dimensions, &output_count); + if (status != LW_STATUS_OK || output_count > input_count) { + return status == LW_STATUS_OK ? LW_STATUS_INVALID_SHAPE : status; + } + /* + * GlobalAveragePool nodes in the bundled models are represented as a + * keep-dimension ReduceMean over NCHW spatial axes. Each output is one + * contiguous input plane, so coordinate decoding is unnecessary. The + * inner loop retains the original row-major addition order exactly. + */ + if (input_rank == 4u && output_rank == 4u && keep_dimensions != 0u && !reduced[0] && + !reduced[1] && reduced[2] && reduced[3] && output_dimensions[0] == input_dimensions[0] && + output_dimensions[1] == input_dimensions[1] && output_dimensions[2] == 1 && + output_dimensions[3] == 1) { + const uint64_t spatial_count = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + uint64_t plane; + for (plane = 0u; plane < output_count; ++plane) { + const uint64_t base = plane * spatial_count; + float sum = 0.0f; + uint64_t spatial_index; + for (spatial_index = 0u; spatial_index < spatial_count; ++spatial_index) { + sum += input[(size_t)(base + spatial_index)]; + } + output[(size_t)plane] = sum / (float)spatial_count; + } + return LW_STATUS_OK; + } + for (axis = output_rank; axis > 0u; --axis) { + output_strides[axis - 1u] = stride; + stride *= (uint32_t)output_dimensions[axis - 1u]; + } + for (input_index = 0u; input_index < output_count; ++input_index) { + output[(size_t)input_index] = 0.0f; + } + for (input_index = 0u; input_index < input_count; ++input_index) { + output[(size_t)output_offset] += input[(size_t)input_index]; + for (axis = input_rank; axis > 0u; --axis) { + uint32_t input_axis = axis - 1u; + uint32_t mapped_axis = input_to_output[input_axis]; + ++coordinates[input_axis]; + if (coordinates[input_axis] < (uint32_t)input_dimensions[input_axis]) { + if (mapped_axis != UINT32_MAX) { + output_offset += output_strides[mapped_axis]; + } + break; + } + coordinates[input_axis] = 0u; + if (mapped_axis != UINT32_MAX) { + output_offset -= + output_strides[mapped_axis] * ((uint32_t)input_dimensions[input_axis] - 1u); + } + } + } + for (input_index = 0u; input_index < output_count; ++input_index) { + output[(size_t)input_index] /= (float)reduction_count; + } + return LW_STATUS_OK; +} + +static int spatial_output(int32_t input, int32_t kernel, int32_t stride, int32_t pad_before, + int32_t pad_after, uint32_t ceil_mode, int32_t* output) { + int64_t numerator; + int64_t value; + if (input <= 0 || kernel <= 0 || stride <= 0 || pad_before < 0 || pad_after < 0) { + return 0; + } + numerator = (int64_t)input + pad_before + pad_after - kernel; + if (numerator < 0) { + return 0; + } + value = (numerator + (ceil_mode != 0u ? stride - 1 : 0)) / stride + 1; + if (ceil_mode != 0u && (value - 1) * stride >= (int64_t)input + pad_before) { + --value; + } + if (value <= 0 || value > INT32_MAX) { + return 0; + } + *output = (int32_t)value; + return 1; +} + +/* + * PP-OCR DET uses one large 2x2, stride-1 SAME_UPPER MaxPool. The general + * pooling loop below must repeatedly derive coordinates and check four bounds + * for every output element. This exact specialization handles the unpadded + * interior with direct row pointers, then writes the right and bottom borders + * separately. Comparisons remain in row-major kernel order so finite values, + * infinities and the existing first-value NaN behavior are preserved. + */ +static void max_pool2x2_stride1_same_upper_f32(const float* input, float* output, + const int32_t dimensions[4]) { + const uint64_t plane_count = (uint64_t)(uint32_t)dimensions[0] * (uint32_t)dimensions[1]; + const uint32_t height = (uint32_t)dimensions[2]; + const uint32_t width = (uint32_t)dimensions[3]; + const size_t plane_elements = (size_t)height * width; + uint64_t plane; + for (plane = 0u; plane < plane_count; ++plane) { + const float* input_plane = input + (size_t)plane * plane_elements; + float* output_plane = output + (size_t)plane * plane_elements; + uint32_t y; + for (y = 0u; y + 1u < height; ++y) { + const float* top_row = input_plane + (size_t)y * width; + const float* bottom_row = top_row + width; + float* output_row = output_plane + (size_t)y * width; + uint32_t x; + for (x = 0u; x + 1u < width; ++x) { + float maximum = top_row[x]; + if (top_row[x + 1u] > maximum) { + maximum = top_row[x + 1u]; + } + if (bottom_row[x] > maximum) { + maximum = bottom_row[x]; + } + if (bottom_row[x + 1u] > maximum) { + maximum = bottom_row[x + 1u]; + } + output_row[x] = maximum; + } + output_row[width - 1u] = top_row[width - 1u]; + if (bottom_row[width - 1u] > output_row[width - 1u]) { + output_row[width - 1u] = bottom_row[width - 1u]; + } + } + { + const float* last_input_row = input_plane + (size_t)(height - 1u) * width; + float* last_output_row = output_plane + (size_t)(height - 1u) * width; + uint32_t x; + for (x = 0u; x + 1u < width; ++x) { + float maximum = last_input_row[x]; + if (last_input_row[x + 1u] > maximum) { + maximum = last_input_row[x + 1u]; + } + last_output_row[x] = maximum; + } + last_output_row[width - 1u] = last_input_row[width - 1u]; + } + } +} + +lw_status lw_scalar_average_pool2d_f32(const float* input, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4], const int32_t kernel[2], + const int32_t strides[2], const int32_t pads[4], + uint32_t ceil_mode, uint32_t count_include_pad) { + uint64_t input_count; + uint64_t output_count; + int32_t expected_height; + int32_t expected_width; + uint32_t batch; + uint32_t channel; + uint32_t output_y; + uint32_t output_x; + lw_status status; + if (input == NULL || output == NULL || input == output || input_dimensions == NULL || + output_dimensions == NULL || kernel == NULL || strides == NULL || pads == NULL) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (ceil_mode > 1u || count_include_pad > 1u) { + return LW_STATUS_INVALID_SHAPE; + } + status = tensor_element_count(4u, input_dimensions, &input_count); + if (status != LW_STATUS_OK) { + return status; + } + status = tensor_element_count(4u, output_dimensions, &output_count); + if (status != LW_STATUS_OK) { + return status; + } + (void)input_count; + (void)output_count; + if (!spatial_output(input_dimensions[2], kernel[0], strides[0], pads[0], pads[2], ceil_mode, + &expected_height) || + !spatial_output(input_dimensions[3], kernel[1], strides[1], pads[1], pads[3], ceil_mode, + &expected_width) || + output_dimensions[0] != input_dimensions[0] || + output_dimensions[1] != input_dimensions[1] || output_dimensions[2] != expected_height || + output_dimensions[3] != expected_width) { + return LW_STATUS_INVALID_SHAPE; + } + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + for (channel = 0u; channel < (uint32_t)input_dimensions[1]; ++channel) { + for (output_y = 0u; output_y < (uint32_t)expected_height; ++output_y) { + int64_t input_y_start = (int64_t)output_y * strides[0] - pads[0]; + for (output_x = 0u; output_x < (uint32_t)expected_width; ++output_x) { + int64_t input_x_start = (int64_t)output_x * strides[1] - pads[1]; + float sum = 0.0f; + uint64_t valid_count = 0u; + int32_t kernel_y; + int32_t kernel_x; + uint64_t output_offset = + (((uint64_t)batch * (uint32_t)input_dimensions[1] + channel) * + (uint32_t)expected_height + + output_y) * + (uint32_t)expected_width + + output_x; + for (kernel_y = 0; kernel_y < kernel[0]; ++kernel_y) { + int64_t input_y = input_y_start + kernel_y; + for (kernel_x = 0; kernel_x < kernel[1]; ++kernel_x) { + int64_t input_x = input_x_start + kernel_x; + if (input_y >= 0 && input_y < input_dimensions[2] && input_x >= 0 && + input_x < input_dimensions[3]) { + uint64_t input_offset = + (((uint64_t)batch * (uint32_t)input_dimensions[1] + channel) * + (uint32_t)input_dimensions[2] + + (uint32_t)input_y) * + (uint32_t)input_dimensions[3] + + (uint32_t)input_x; + sum += input[(size_t)input_offset]; + ++valid_count; + } + } + } + if (count_include_pad != 0u) { + valid_count = (uint64_t)(uint32_t)kernel[0] * (uint32_t)kernel[1]; + } + if (valid_count == 0u) { + return LW_STATUS_INVALID_SHAPE; + } + output[(size_t)output_offset] = sum / (float)valid_count; + } + } + } + } + return LW_STATUS_OK; +} + +lw_status lw_scalar_max_pool2d_f32(const float* input, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4], const int32_t kernel[2], + const int32_t strides[2], const int32_t pads[4], + uint32_t ceil_mode) { + uint64_t element_count; + int32_t expected_height; + int32_t expected_width; + uint32_t batch; + uint32_t channel; + uint32_t output_y; + uint32_t output_x; + lw_status status; + if (input == NULL || output == NULL || input == output || input_dimensions == NULL || + output_dimensions == NULL || kernel == NULL || strides == NULL || pads == NULL) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (ceil_mode > 1u) { + return LW_STATUS_INVALID_SHAPE; + } + status = tensor_element_count(4u, input_dimensions, &element_count); + if (status != LW_STATUS_OK) { + return status; + } + status = tensor_element_count(4u, output_dimensions, &element_count); + if (status != LW_STATUS_OK) { + return status; + } + if (!spatial_output(input_dimensions[2], kernel[0], strides[0], pads[0], pads[2], ceil_mode, + &expected_height) || + !spatial_output(input_dimensions[3], kernel[1], strides[1], pads[1], pads[3], ceil_mode, + &expected_width) || + output_dimensions[0] != input_dimensions[0] || + output_dimensions[1] != input_dimensions[1] || output_dimensions[2] != expected_height || + output_dimensions[3] != expected_width) { + return LW_STATUS_INVALID_SHAPE; + } + if (kernel[0] == 2 && kernel[1] == 2 && strides[0] == 1 && strides[1] == 1 && + pads[0] == 0 && pads[1] == 0 && pads[2] == 1 && pads[3] == 1 && ceil_mode == 0u && + output_dimensions[2] == input_dimensions[2] && + output_dimensions[3] == input_dimensions[3]) { + max_pool2x2_stride1_same_upper_f32(input, output, input_dimensions); + return LW_STATUS_OK; + } + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + for (channel = 0u; channel < (uint32_t)input_dimensions[1]; ++channel) { + for (output_y = 0u; output_y < (uint32_t)expected_height; ++output_y) { + int64_t input_y_start = (int64_t)output_y * strides[0] - pads[0]; + for (output_x = 0u; output_x < (uint32_t)expected_width; ++output_x) { + int64_t input_x_start = (int64_t)output_x * strides[1] - pads[1]; + float maximum = -INFINITY; + int found = 0; + int32_t kernel_y; + int32_t kernel_x; + uint64_t output_offset = + (((uint64_t)batch * (uint32_t)input_dimensions[1] + channel) * + (uint32_t)expected_height + + output_y) * + (uint32_t)expected_width + + output_x; + for (kernel_y = 0; kernel_y < kernel[0]; ++kernel_y) { + int64_t input_y = input_y_start + kernel_y; + for (kernel_x = 0; kernel_x < kernel[1]; ++kernel_x) { + int64_t input_x = input_x_start + kernel_x; + if (input_y >= 0 && input_y < input_dimensions[2] && input_x >= 0 && + input_x < input_dimensions[3]) { + uint64_t input_offset = + (((uint64_t)batch * (uint32_t)input_dimensions[1] + channel) * + (uint32_t)input_dimensions[2] + + (uint32_t)input_y) * + (uint32_t)input_dimensions[3] + + (uint32_t)input_x; + float value = input[(size_t)input_offset]; + if (!found || value > maximum) { + maximum = value; + } + found = 1; + } + } + } + if (!found) { + return LW_STATUS_INVALID_SHAPE; + } + output[(size_t)output_offset] = maximum; + } + } + } + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_softmax.c b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_softmax.c new file mode 100644 index 00000000..db0e08d4 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/kernels/scalar_softmax.c @@ -0,0 +1,191 @@ +#include "scalar_kernels.h" +#include "cpu_features.h" +#include "simd_kernels.h" + +/* Numerically stable softmax: subtract the row maximum before exponentiation. */ + +#include +#include +#include + +lw_status lw_scalar_softmax_f32(const float* input, float* output, uint32_t rank, + const int32_t* dimensions, int32_t axis) { + uint64_t outer_count = 1u; + uint64_t inner_count = 1u; + uint64_t axis_count; + uint64_t outer; + uint64_t inner; + uint32_t normalized_axis; + uint32_t index; + + if (input == NULL || output == NULL) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (rank == 0u || rank > LW_MAX_DIMS || dimensions == NULL || axis < -(int32_t)rank || + axis >= (int32_t)rank) { + return LW_STATUS_INVALID_SHAPE; + } + normalized_axis = axis < 0 ? (uint32_t)((int32_t)rank + axis) : (uint32_t)axis; + for (index = 0u; index < rank; ++index) { + uint64_t dimension; + if (dimensions[index] <= 0) { + return LW_STATUS_INVALID_SHAPE; + } + dimension = (uint32_t)dimensions[index]; + if (index < normalized_axis) { + if (outer_count > UINT64_MAX / dimension) { + return LW_STATUS_OUT_OF_BOUNDS; + } + outer_count *= dimension; + } else if (index > normalized_axis) { + if (inner_count > UINT64_MAX / dimension) { + return LW_STATUS_OUT_OF_BOUNDS; + } + inner_count *= dimension; + } + } + axis_count = (uint32_t)dimensions[normalized_axis]; + if (outer_count > UINT64_MAX / axis_count || + outer_count * axis_count > UINT64_MAX / inner_count || + outer_count * axis_count * inner_count > (uint64_t)(SIZE_MAX / sizeof(float))) { + return LW_STATUS_OUT_OF_BOUNDS; + } + + /* The REC classifier axis is the contiguous final dimension. Direct row + * pointers remove the general strided offset arithmetic while preserving + * the maximum, expf, accumulation and division order exactly. */ + if (inner_count == 1u) { + if (axis_count >= 256u && + lw_simd_level_is_avx2(lw_detect_simd_level())) { + lw_avx2_softmax_contiguous_f32(input, output, outer_count, axis_count); + return LW_STATUS_OK; + } + for (outer = 0u; outer < outer_count; ++outer) { + const float* input_row = input + (size_t)(outer * axis_count); + float* output_row = output + (size_t)(outer * axis_count); + float maximum = input_row[0]; + float sum = 0.0f; + uint64_t axis_index; + for (axis_index = 1u; axis_index < axis_count; ++axis_index) { + if (input_row[(size_t)axis_index] > maximum) { + maximum = input_row[(size_t)axis_index]; + } + } + for (axis_index = 0u; axis_index < axis_count; ++axis_index) { + float value = expf(input_row[(size_t)axis_index] - maximum); + output_row[(size_t)axis_index] = value; + sum += value; + } + for (axis_index = 0u; axis_index < axis_count; ++axis_index) { + output_row[(size_t)axis_index] /= sum; + } + } + return LW_STATUS_OK; + } + + for (outer = 0u; outer < outer_count; ++outer) { + for (inner = 0u; inner < inner_count; ++inner) { + uint64_t axis_index; + uint64_t base = outer * axis_count * inner_count + inner; + /* Shifting by the maximum preserves the result and prevents expf + * from overflowing on large logits. */ + float maximum = input[(size_t)base]; + float sum = 0.0f; + for (axis_index = 1u; axis_index < axis_count; ++axis_index) { + float value = input[(size_t)(base + axis_index * inner_count)]; + if (value > maximum) { + maximum = value; + } + } + for (axis_index = 0u; axis_index < axis_count; ++axis_index) { + size_t offset = (size_t)(base + axis_index * inner_count); + float value = expf(input[offset] - maximum); + output[offset] = value; + sum += value; + } + for (axis_index = 0u; axis_index < axis_count; ++axis_index) { + size_t offset = (size_t)(base + axis_index * inner_count); + output[offset] /= sum; + } + } + } + return LW_STATUS_OK; +} + +lw_status lw_ctc_greedy_softmax_contiguous_f32(const float* input, uint32_t* best_indices, + float* emitted_probabilities, + uint64_t row_count, uint64_t axis_count) { + uint64_t row; + if (input == NULL || best_indices == NULL || emitted_probabilities == NULL || row_count == 0u || + axis_count == 0u || axis_count > UINT32_MAX || + row_count > (uint64_t)(SIZE_MAX / sizeof(*best_indices)) || + row_count > (uint64_t)(SIZE_MAX / sizeof(*emitted_probabilities)) || + row_count > UINT64_MAX / axis_count || + row_count * axis_count > (uint64_t)(SIZE_MAX / sizeof(float))) { + return LW_STATUS_INVALID_ARGUMENT; + } + /* Argmax is required for every time step, but the CTC confidence score + * consumes Softmax only for non-blank, non-repeated emitted classes. */ + for (row = 0u; row < row_count; ++row) { + const float* input_row = input + (size_t)(row * axis_count); + uint32_t best_index = 0u; + float maximum = input_row[0]; + uint64_t index; + if (!isfinite(maximum)) { + return LW_STATUS_INVALID_ARGUMENT; + } + for (index = 1u; index < axis_count; ++index) { + float value = input_row[(size_t)index]; + if (!isfinite(value)) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (value > maximum) { + maximum = value; + best_index = (uint32_t)index; + } + } + best_indices[(size_t)row] = best_index; + } + return lw_ctc_emitted_softmax_contiguous_f32(input, best_indices, + emitted_probabilities, row_count, axis_count); +} + +lw_status lw_ctc_emitted_softmax_contiguous_f32(const float* input, + const uint32_t* best_indices, + float* emitted_probabilities, + uint64_t row_count, uint64_t axis_count) { + uint64_t row; + if (input == NULL || best_indices == NULL || emitted_probabilities == NULL || + row_count == 0u || axis_count == 0u || axis_count > UINT32_MAX || + row_count > (uint64_t)(SIZE_MAX / sizeof(*best_indices)) || + row_count > (uint64_t)(SIZE_MAX / sizeof(*emitted_probabilities)) || + row_count > UINT64_MAX / axis_count || + row_count * axis_count > (uint64_t)(SIZE_MAX / sizeof(float))) { + return LW_STATUS_INVALID_ARGUMENT; + } + for (row = 0u; row < row_count; ++row) { + if (best_indices[(size_t)row] >= axis_count) { + return LW_STATUS_INVALID_ARGUMENT; + } + emitted_probabilities[(size_t)row] = 0.0f; + } + if (axis_count >= 256u && lw_simd_level_is_avx2(lw_detect_simd_level())) { + lw_avx2_ctc_emitted_softmax_contiguous_f32(input, best_indices, + emitted_probabilities, row_count, axis_count); + return LW_STATUS_OK; + } + for (row = 0u; row < row_count; ++row) { + uint32_t best_index = best_indices[(size_t)row]; + if (best_index != 0u && (row == 0u || best_index != best_indices[(size_t)row - 1u])) { + const float* input_row = input + (size_t)(row * axis_count); + float maximum = input_row[best_index]; + float sum = 0.0f; + uint64_t index; + for (index = 0u; index < axis_count; ++index) { + sum += expf(input_row[(size_t)index] - maximum); + } + emitted_probabilities[(size_t)row] = 1.0f / sum; + } + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/classifier.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/classifier.c new file mode 100644 index 00000000..48483841 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/classifier.c @@ -0,0 +1,436 @@ +#include "lw_infer.h" +#include "abi_compat_internal.h" + +/* Public CLS handle for detecting whether a text crop needs 180-degree rotation. */ + +#include "cls_internal.h" +#include "error_internal.h" +#include "executor_internal.h" +#include "profile_internal.h" + +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +#include "x64_rec_backend_internal.h" +#endif + +#include +#include +#include +#include +#include +#include + +#define LW_CLS_DEFAULT_MAX_IMAGE_PIXELS UINT64_C(40000000) + +struct lw_classifier { + lw_model* model; + lw_session* session; + float* input; + float probabilities[LW_CLS_CLASS_COUNT]; + uint64_t input_element_count; + uint64_t max_image_pixels; + lw_classifier_info info; + lw_cls_preprocess_workspace preprocess_workspace; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + /* Compiled x64 backbone (reuses the REC lowering with the CLS input + * shape and no CTC tail). NULL when unsupported; the canonical session + * above remains the fallback. */ + lw_x64_rec_program* x64_program; + lw_x64_rec_instance* x64_instance; +#endif +}; + +static void clear_result(lw_classification_result* result) { + memset(result, 0, sizeof(*result)); + result->struct_size = (uint32_t)sizeof(*result); +} + +void lw_classifier_options_init(lw_classifier_options* options) { + lw_model_options model_options; + lw_session_options session_options; + if (options == NULL) { + return; + } + lw_model_options_init(&model_options); + lw_session_options_init(&session_options); + memset(options, 0, sizeof(*options)); + options->struct_size = (uint32_t)sizeof(*options); + options->max_model_file_size = model_options.max_file_size; + options->max_workspace_size = session_options.max_workspace_size; + options->max_tensor_size = session_options.max_tensor_size; + options->max_image_pixels = LW_CLS_DEFAULT_MAX_IMAGE_PIXELS; +} + +void lw_classifier_info_init(lw_classifier_info* info) { + if (info == NULL) { + return; + } + memset(info, 0, sizeof(*info)); + info->struct_size = (uint32_t)sizeof(*info); +} + +void lw_classification_result_init(lw_classification_result* result) { + if (result == NULL) { + return; + } + clear_result(result); +} + +static lw_status validate_options(const lw_classifier_options* options, uint64_t* max_image_pixels, + lw_model_options* model_options, + lw_session_options* session_options, lw_error* error) { + lw_classifier_options defaults; + lw_classifier_options values; + lw_classifier_options_init(&defaults); + values = defaults; + *max_image_pixels = defaults.max_image_pixels; + lw_model_options_init(model_options); + lw_session_options_init(session_options); + if (options == NULL) { + return LW_STATUS_OK; + } + if (!lw_abi_copy_input_prefix(&values, sizeof(values), options)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid classifier options structure"); + return LW_STATUS_INVALID_ARGUMENT; + } + values.struct_size = (uint32_t)sizeof(values); + if (values.reserved != 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid classifier options structure"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (values.max_model_file_size != 0u) { + model_options->max_file_size = values.max_model_file_size; + } + if (values.max_workspace_size != 0u) { + session_options->max_workspace_size = values.max_workspace_size; + } + if (values.max_tensor_size != 0u) { + session_options->max_tensor_size = values.max_tensor_size; + } + if (values.max_image_pixels != 0u) { + *max_image_pixels = values.max_image_pixels; + } + return LW_STATUS_OK; +} + +lw_status lw_classifier_create(const char* model_path_utf8, const lw_classifier_options* options, + lw_classifier** out_classifier, lw_error* error) { + lw_model_options model_options; + lw_session_options session_options; + lw_tensor_desc input_desc; + lw_tensor_desc output_desc; + lw_session_info session_info; + lw_classifier* classifier = NULL; + uint64_t max_image_pixels; + lw_status status; + if (out_classifier != NULL) { + *out_classifier = NULL; + } + if (model_path_utf8 == NULL || model_path_utf8[0] == '\0' || out_classifier == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "model path and output classifier are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + status = validate_options(options, &max_image_pixels, &model_options, &session_options, error); + if (status != LW_STATUS_OK) { + return status; + } + classifier = (lw_classifier*)calloc(1u, sizeof(*classifier)); + if (classifier == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate classifier handle"); + return LW_STATUS_OUT_OF_MEMORY; + } + classifier->max_image_pixels = max_image_pixels; + status = lw_model_load(model_path_utf8, &model_options, &classifier->model, error); + if (status != LW_STATUS_OK) { + goto fail; + } + lw_tensor_desc_init(&input_desc); + input_desc.dtype = LW_DTYPE_F32; + input_desc.rank = 4u; + input_desc.dimensions[0] = 1; + input_desc.dimensions[1] = 3; + input_desc.dimensions[2] = (int32_t)LW_CLS_INPUT_HEIGHT; + input_desc.dimensions[3] = (int32_t)LW_CLS_INPUT_WIDTH; + status = lw_session_create(classifier->model, &input_desc, 1u, &session_options, + &classifier->session, error); + if (status != LW_STATUS_OK) { + goto fail; + } + lw_tensor_desc_init(&output_desc); + status = lw_session_get_output_desc(classifier->session, 0u, &output_desc); + if (status != LW_STATUS_OK || output_desc.dtype != LW_DTYPE_F32 || output_desc.rank != 2u || + output_desc.dimensions[0] != 1 || + output_desc.dimensions[1] != (int32_t)LW_CLS_CLASS_COUNT) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "classifier model output must have shape [1,2]"); + status = LW_STATUS_INVALID_SHAPE; + goto fail; + } + classifier->input_element_count = (uint64_t)3u * LW_CLS_INPUT_HEIGHT * LW_CLS_INPUT_WIDTH; + if (classifier->input_element_count > SIZE_MAX / sizeof(*classifier->input)) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "classifier input buffer size overflows"); + status = LW_STATUS_OUT_OF_BOUNDS; + goto fail; + } + classifier->input = + (float*)malloc((size_t)classifier->input_element_count * sizeof(*classifier->input)); + if (classifier->input == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate classifier input buffer"); + status = LW_STATUS_OUT_OF_MEMORY; + goto fail; + } + lw_session_info_init(&session_info); + status = lw_session_get_info(classifier->session, &session_info); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "unable to read classifier session information"); + goto fail; + } + lw_classifier_info_init(&classifier->info); + classifier->info.input_width = LW_CLS_INPUT_WIDTH; + classifier->info.input_height = LW_CLS_INPUT_HEIGHT; + classifier->info.class_count = LW_CLS_CLASS_COUNT; + classifier->info.workspace_size = session_info.workspace_size; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + { + lw_error backend_error; + lw_error_init(&backend_error); + if (lw_x64_rec_backend_compile_input(classifier->model, LW_CLS_INPUT_HEIGHT, + LW_CLS_INPUT_WIDTH, LW_X64_REC_COMPILE_NHWC, 0u, + &classifier->x64_program, + &backend_error) != LW_X64_REC_COMPILE_OK || + classifier->x64_program == NULL || + classifier->x64_program->unsupported_nodes != 0u || + lw_x64_rec_instance_create(classifier->x64_program, &classifier->x64_instance, + &backend_error) != LW_STATUS_OK) { + lw_x64_rec_instance_free(classifier->x64_instance); + lw_x64_rec_program_free(classifier->x64_program); + classifier->x64_instance = NULL; + classifier->x64_program = NULL; + } +#if defined(__EMSCRIPTEN__) + if (classifier->x64_program != NULL && classifier->x64_instance != NULL) { + uint64_t packed_bytes = 0u; + uint32_t constant_index; + for (constant_index = 0u; + constant_index < classifier->x64_program->packed_constant_count; + ++constant_index) { + packed_bytes += classifier->x64_program->constants[constant_index].bytes; + } + (void)fprintf(stderr, + "LW_WASM_COMPILED_CLS layout=nhwc ops=%u unsupported=%u " + "arena_bytes=%llu packed_bytes=%llu\n", + classifier->x64_program->op_count, + classifier->x64_program->unsupported_nodes, + (unsigned long long)classifier->x64_program->arena_bytes, + (unsigned long long)packed_bytes); + } +#endif + } +#endif + *out_classifier = classifier; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + +fail: + lw_classifier_free(classifier); + return status; +} + +void lw_classifier_free(lw_classifier* classifier) { + if (classifier == NULL) { + return; + } +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_instance_free(classifier->x64_instance); + lw_x64_rec_program_free(classifier->x64_program); +#endif + free(classifier->input); + lw_session_free(classifier->session); + lw_model_free(classifier->model); + lw_cls_preprocess_workspace_free(&classifier->preprocess_workspace); + free(classifier); +} + +lw_status lw_classifier_get_info(const lw_classifier* classifier, lw_classifier_info* info) { + if (classifier == NULL || info == NULL || + !lw_abi_copy_output_prefix(info, info->struct_size, &classifier->info, + sizeof(classifier->info))) { + return LW_STATUS_INVALID_ARGUMENT; + } + return LW_STATUS_OK; +} + +static lw_status classifier_classify_bgr_u8_impl(lw_classifier* classifier, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + lw_classification_result* result, + lw_pipeline_component_profile* profile, + lw_error* error) { + uint64_t source_pixels; + uint32_t resized_width = 0u; + uint32_t label; + lw_classification_result output; + uint32_t result_size; + uint64_t started; + lw_status status; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + uint64_t backend_input_count = 0u; + float* backend_input = classifier == NULL || classifier->x64_instance == NULL + ? NULL : lw_x64_rec_instance_input(classifier->x64_instance, &backend_input_count); +#if defined(LW_EXPERIMENTAL_CLS_FIXED_POINT_RESIZE) + int direct_nhwc_input = 0; +#endif +#endif + if (classifier == NULL || source == NULL || result == NULL || + result->struct_size < sizeof(uint32_t)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "classifier, BGR source, and initialized result are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + result_size = result->struct_size; + clear_result(&output); + if (source_width == 0u || source_height == 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "source image dimensions must be positive"); + return LW_STATUS_INVALID_ARGUMENT; + } + source_pixels = (uint64_t)source_width * source_height; + if (source_pixels > classifier->max_image_pixels) { + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, "source image exceeds max_image_pixels"); + return LW_STATUS_MEMORY_LIMIT; + } + started = lw_pipeline_profile_now(profile); +#if defined(LW_EXPERIMENTAL_CLS_FIXED_POINT_RESIZE) +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + if (backend_input != NULL && backend_input_count == classifier->input_element_count) { + status = lw_cls_preprocess_bgr_u8_fixed_nhwc( + source, source_byte_count, source_width, source_height, source_stride, + backend_input, backend_input_count, &resized_width, + &classifier->preprocess_workspace); + direct_nhwc_input = 1; + } else +#endif + status = lw_cls_preprocess_bgr_u8_fixed(source, source_byte_count, source_width, + source_height, source_stride, classifier->input, + classifier->input_element_count, &resized_width, + &classifier->preprocess_workspace); +#else + status = lw_cls_preprocess_bgr_u8(source, source_byte_count, source_width, source_height, + source_stride, classifier->input, + classifier->input_element_count, &resized_width); +#endif + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "BGR source layout is invalid"); + return status; + } + lw_pipeline_profile_add_elapsed(profile == NULL ? NULL : &profile->preprocess_nanoseconds, + started, profile); + started = lw_pipeline_profile_now(profile); + status = LW_STATUS_UNSUPPORTED; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + if (classifier->x64_instance != NULL && backend_input != NULL && + backend_input_count == classifier->input_element_count) { +#if defined(LW_EXPERIMENTAL_CLS_FIXED_POINT_RESIZE) + if (!direct_nhwc_input) { +#endif + /* NCHW planes -> interleaved NHWC; pure data movement, bit-exact. */ + const uint32_t height = LW_CLS_INPUT_HEIGHT; + const uint32_t width = LW_CLS_INPUT_WIDTH; + uint32_t y; + uint32_t x; + for (y = 0u; y < height; ++y) { + for (x = 0u; x < width; ++x) { + uint32_t channel; + for (channel = 0u; channel < 3u; ++channel) { + backend_input[((size_t)y * width + x) * 3u + channel] = + classifier->input[((size_t)channel * height + y) * width + x]; + } + } + } +#if defined(LW_EXPERIMENTAL_CLS_FIXED_POINT_RESIZE) + } +#endif + status = lw_x64_rec_instance_run(classifier->x64_instance, error); + if (status == LW_STATUS_OK) { + if (profile != NULL) + lw_profile_add_value(&profile->compiled_backend_runs, 1u); + const lw_x64_rec_program* program = classifier->x64_program; + const float* probabilities = + (const float*)(const void*)(classifier->x64_instance->arena + + (size_t)program->values[program->output_value] + .offset); + classifier->probabilities[0] = probabilities[0]; + classifier->probabilities[1] = probabilities[1]; + } + } +#endif + if (status == LW_STATUS_UNSUPPORTED) { +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) && defined(LW_EXPERIMENTAL_CLS_FIXED_POINT_RESIZE) + if (direct_nhwc_input) { + /* Recreate canonical NCHW input before the session fallback. */ + status = lw_cls_preprocess_bgr_u8_fixed( + source, source_byte_count, source_width, source_height, source_stride, + classifier->input, classifier->input_element_count, &resized_width, + &classifier->preprocess_workspace); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "CLS fallback preprocessing failed"); + return status; + } + } +#endif + status = profile == NULL + ? lw_execute_session_f32(classifier->session, classifier->input, + classifier->input_element_count, + classifier->probabilities, LW_CLS_CLASS_COUNT, error) + : lw_execute_session_f32_profiled( + classifier->session, classifier->input, classifier->input_element_count, + classifier->probabilities, LW_CLS_CLASS_COUNT, &profile->execution, + error); + if (status == LW_STATUS_OK && profile != NULL) + lw_profile_add_value(&profile->canonical_fallback_runs, 1u); + } + lw_pipeline_profile_add_elapsed(profile == NULL ? NULL : &profile->graph_nanoseconds, started, + profile); + if (status != LW_STATUS_OK) { + return status; + } + started = lw_pipeline_profile_now(profile); + if (!isfinite(classifier->probabilities[0]) || !isfinite(classifier->probabilities[1])) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "classifier output contains a non-finite value"); + return LW_STATUS_INVALID_ARGUMENT; + } + label = classifier->probabilities[1] > classifier->probabilities[0] ? 1u : 0u; + output.label = label; + output.score = classifier->probabilities[label]; + output.resized_width = resized_width; + output.orientation_degrees = label == 0u ? 0u : 180u; + lw_pipeline_profile_add_elapsed(profile == NULL ? NULL : &profile->postprocess_nanoseconds, + started, profile); + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +lw_status lw_classifier_classify_bgr_u8(lw_classifier* classifier, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + lw_classification_result* result, lw_error* error) { + return classifier_classify_bgr_u8_impl(classifier, source, source_byte_count, source_width, + source_height, source_stride, result, NULL, error); +} + +lw_status lw_classifier_classify_bgr_u8_profiled(lw_classifier* classifier, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + lw_classification_result* result, + lw_pipeline_component_profile* profile, + lw_error* error) { + if (profile == NULL || profile->execution.struct_size != sizeof(profile->execution) || + profile->execution.clock == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "an initialized classifier profile and clock are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + return classifier_classify_bgr_u8_impl(classifier, source, source_byte_count, source_width, + source_height, source_stride, result, profile, error); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/cls_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/cls_internal.h new file mode 100644 index 00000000..eb6bbdfa --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/cls_internal.h @@ -0,0 +1,48 @@ +#ifndef LW_CLS_INTERNAL_H +#define LW_CLS_INTERNAL_H + +/* Private classifier preprocessing contract. */ + +#include "lw_infer.h" + +#include + +#define LW_CLS_INPUT_HEIGHT 80u +#define LW_CLS_INPUT_WIDTH 160u +#define LW_CLS_CLASS_COUNT 2u +#define LW_CLS_SOURCE_WINDOW_MAX_WIDTH_PER_HEIGHT 4u + +lw_status lw_cls_preprocess_bgr_u8(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, float* output, + uint64_t output_element_count, uint32_t* resized_width); + +/* Reference keep-aspect variant: resize the left 4H window to + * min(160, ceil(80*w/h)) x 80 with the OpenCV-parity fixed-point path, pad + * unused trailing columns with -1, and normalize through the RecNorm LUT. */ +typedef struct lw_cls_preprocess_workspace { + int32_t* x_offsets; + int16_t* x_coefficients; + int32_t* row0; + int32_t* row1; + uint32_t width; +} lw_cls_preprocess_workspace; + +lw_status lw_cls_preprocess_bgr_u8_fixed( + const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, float* output, + uint64_t output_element_count, uint32_t* resized_width, + lw_cls_preprocess_workspace* workspace); + +/* Same fixed-point samples and LUT as the NCHW variant, directly interleaved + * into the compiled backend's NHWC input arena. */ +lw_status lw_cls_preprocess_bgr_u8_fixed_nhwc( + const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, float* output, + uint64_t output_element_count, uint32_t* resized_width, + lw_cls_preprocess_workspace* workspace); + +void lw_cls_preprocess_workspace_init(lw_cls_preprocess_workspace* workspace); +void lw_cls_preprocess_workspace_free(lw_cls_preprocess_workspace* workspace); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/cls_preprocess.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/cls_preprocess.c new file mode 100644 index 00000000..6b81aff4 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/cls_preprocess.c @@ -0,0 +1,272 @@ +#include "cls_internal.h" +#include "resize_fixed_internal.h" + +/* Resize the left 4H crop window and normalize it for the direction classifier. */ + +#include +#include +#include +#include +#include + +static uint32_t clamp_coordinate(int64_t coordinate, uint32_t limit) { + if (coordinate < 0) { + return 0u; + } + if ((uint64_t)coordinate >= limit) { + return limit - 1u; + } + return (uint32_t)coordinate; +} + +lw_status lw_cls_preprocess_bgr_u8(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, float* output, + uint64_t output_element_count, uint32_t* resized_width) { + const uint64_t required_output_elements = + (uint64_t)3u * LW_CLS_INPUT_HEIGHT * LW_CLS_INPUT_WIDTH; + const double normalize_scale = 2.0 / 255.0; + uint64_t row_bytes; + uint64_t required_source_bytes; + uint64_t source_window_width; + uint64_t channel_plane; + uint32_t channel; + uint32_t output_y; + + if (resized_width != NULL) { + *resized_width = 0u; + } + if (source == NULL || output == NULL || source_width == 0u || source_height == 0u) { + return LW_STATUS_INVALID_ARGUMENT; + } + row_bytes = (uint64_t)source_width * 3u; + if (row_bytes > UINT32_MAX || source_stride < row_bytes || + (uint64_t)(source_height - 1u) * source_stride > UINT64_MAX - row_bytes) { + return LW_STATUS_INVALID_SHAPE; + } + required_source_bytes = (uint64_t)(source_height - 1u) * source_stride + row_bytes; + if (required_source_bytes > SIZE_MAX || source_byte_count < required_source_bytes || + output_element_count != required_output_elements || + required_output_elements > SIZE_MAX / sizeof(float)) { + return LW_STATUS_INVALID_SHAPE; + } + source_window_width = (uint64_t)source_height * LW_CLS_SOURCE_WINDOW_MAX_WIDTH_PER_HEIGHT; + if (source_window_width > source_width) { + source_window_width = source_width; + } + if (source_window_width == 0u) { + return LW_STATUS_INVALID_SHAPE; + } + channel_plane = (uint64_t)LW_CLS_INPUT_HEIGHT * LW_CLS_INPUT_WIDTH; + + for (output_y = 0u; output_y < LW_CLS_INPUT_HEIGHT; ++output_y) { + const double source_y = + ((double)output_y + 0.5) * source_height / LW_CLS_INPUT_HEIGHT - 0.5; + const int64_t source_y0_raw = (int64_t)floor(source_y); + const int64_t source_y1_raw = source_y0_raw + 1; + const uint32_t source_y0 = clamp_coordinate(source_y0_raw, source_height); + const uint32_t source_y1 = clamp_coordinate(source_y1_raw, source_height); + const double weight_y = source_y - (double)source_y0_raw; + uint32_t output_x; + for (output_x = 0u; output_x < LW_CLS_INPUT_WIDTH; ++output_x) { + const double source_x = + ((double)output_x + 0.5) * source_window_width / LW_CLS_INPUT_WIDTH - 0.5; + const int64_t source_x0_raw = (int64_t)floor(source_x); + const int64_t source_x1_raw = source_x0_raw + 1; + const uint32_t source_x0 = clamp_coordinate(source_x0_raw, (uint32_t)source_window_width); + const uint32_t source_x1 = clamp_coordinate(source_x1_raw, (uint32_t)source_window_width); + const double weight_x = source_x - (double)source_x0_raw; + for (channel = 0u; channel < 3u; ++channel) { + const double top_left = source[(size_t)((uint64_t)source_y0 * source_stride + + (uint64_t)source_x0 * 3u + channel)]; + const double top_right = source[(size_t)((uint64_t)source_y0 * source_stride + + (uint64_t)source_x1 * 3u + channel)]; + const double bottom_left = + source[(size_t)((uint64_t)source_y1 * source_stride + + (uint64_t)source_x0 * 3u + channel)]; + const double bottom_right = + source[(size_t)((uint64_t)source_y1 * source_stride + + (uint64_t)source_x1 * 3u + channel)]; + const double top = top_left + (top_right - top_left) * weight_x; + const double bottom = bottom_left + (bottom_right - bottom_left) * weight_x; + const double value = top + (bottom - top) * weight_y; + const uint64_t output_index = + (uint64_t)channel * channel_plane + + (uint64_t)output_y * LW_CLS_INPUT_WIDTH + output_x; + output[(size_t)output_index] = (float)(value * normalize_scale - 1.0); + } + } + } + if (resized_width != NULL) { + *resized_width = LW_CLS_INPUT_WIDTH; + } + return LW_STATUS_OK; +} + +static void cls_fixed_workspace_ensure(lw_cls_preprocess_workspace* workspace, + uint32_t width) { + if (workspace->width != width) { + int32_t* x_offsets = (int32_t*)malloc((size_t)width * sizeof(*x_offsets)); + int16_t* x_coefficients = (int16_t*)malloc((size_t)width * 2u * sizeof(*x_coefficients)); + int32_t* row0 = (int32_t*)malloc((size_t)width * 3u * sizeof(*row0)); + int32_t* row1 = (int32_t*)malloc((size_t)width * 3u * sizeof(*row1)); + if (x_offsets == NULL || x_coefficients == NULL || row0 == NULL || row1 == NULL) { + free(row1); + free(row0); + free(x_coefficients); + free(x_offsets); + return; + } + free(workspace->row1); + free(workspace->row0); + free(workspace->x_coefficients); + free(workspace->x_offsets); + workspace->x_offsets = x_offsets; + workspace->x_coefficients = x_coefficients; + workspace->row0 = row0; + workspace->row1 = row1; + workspace->width = width; + } +} + +void lw_cls_preprocess_workspace_init(lw_cls_preprocess_workspace* workspace) { + if (workspace != NULL) memset(workspace, 0, sizeof(*workspace)); +} + +void lw_cls_preprocess_workspace_free(lw_cls_preprocess_workspace* workspace) { + if (workspace == NULL) return; + free(workspace->row1); + free(workspace->row0); + free(workspace->x_coefficients); + free(workspace->x_offsets); + memset(workspace, 0, sizeof(*workspace)); +} + +static lw_status cls_preprocess_bgr_u8_fixed_layout( + const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, float* output, + uint64_t output_element_count, uint32_t* resized_width, + lw_cls_preprocess_workspace* workspace, int nhwc) { + const uint64_t required_output_elements = + (uint64_t)3u * LW_CLS_INPUT_HEIGHT * LW_CLS_INPUT_WIDTH; + uint64_t row_bytes; + uint64_t required_source_bytes; + uint32_t window_width; + uint32_t actual_width; + uint32_t channel_plane; + uint32_t oy; + uint32_t ox; + uint32_t channel; + float rec_lut[256]; + if (resized_width != NULL) { + *resized_width = 0u; + } + if (source == NULL || output == NULL || workspace == NULL || source_width == 0u || + source_height == 0u) { + return LW_STATUS_INVALID_ARGUMENT; + } + row_bytes = (uint64_t)source_width * 3u; + if (row_bytes > UINT32_MAX || source_stride < row_bytes || + (uint64_t)(source_height - 1u) * source_stride > UINT64_MAX - row_bytes) { + return LW_STATUS_INVALID_SHAPE; + } + required_source_bytes = (uint64_t)(source_height - 1u) * source_stride + row_bytes; + if (required_source_bytes > SIZE_MAX || source_byte_count < required_source_bytes || + output_element_count != required_output_elements || + required_output_elements > SIZE_MAX / sizeof(float)) { + return LW_STATUS_INVALID_SHAPE; + } + /* Left 4:1 window on the original buffer (no allocation, stride + * unchanged), then keep-aspect to height 80 with trailing -1 columns. */ + window_width = source_width; + if ((uint64_t)LW_CLS_INPUT_HEIGHT * window_width > + (uint64_t)(LW_CLS_INPUT_WIDTH * 2u) * source_height) { + window_width = (uint32_t)(((uint64_t)(LW_CLS_INPUT_WIDTH * 2u) * source_height) / + LW_CLS_INPUT_HEIGHT); + } + if (window_width == 0u) { + return LW_STATUS_INVALID_SHAPE; + } + actual_width = (uint32_t)(((uint64_t)LW_CLS_INPUT_HEIGHT * window_width + + source_height - 1u) / + source_height); + if (actual_width > LW_CLS_INPUT_WIDTH) actual_width = LW_CLS_INPUT_WIDTH; + if (actual_width < LW_CLS_INPUT_WIDTH) { + uint64_t count = required_output_elements; + uint64_t i; + for (i = 0u; i < count; ++i) output[(size_t)i] = -1.0f; + } + channel_plane = (uint64_t)LW_CLS_INPUT_HEIGHT * LW_CLS_INPUT_WIDTH; + cls_fixed_workspace_ensure(workspace, actual_width); + if (workspace->x_offsets == NULL || workspace->row0 == NULL) { + return LW_STATUS_OUT_OF_MEMORY; + } + lw_fixed_build_linear_coefficients(window_width, actual_width, workspace->x_offsets, + workspace->x_coefficients); + for (ox = 0u; ox < 256u; ++ox) { + rec_lut[ox] = (float)((double)ox * (2.0 / 255.0) - 1.0); + } + for (oy = 0u; oy < LW_CLS_INPUT_HEIGHT; ++oy) { + int32_t source_y; + int16_t beta0; + int16_t beta1; + uint32_t source_y0; + uint32_t source_y1; + uint32_t destination = oy * LW_CLS_INPUT_WIDTH; + lw_fixed_get_linear_coordinate(oy, source_height, LW_CLS_INPUT_HEIGHT, &source_y, + &beta0, &beta1); + source_y0 = source_y < 0 + ? 0u + : ((uint32_t)source_y >= source_height ? source_height - 1u + : (uint32_t)source_y); + { + int32_t clamped = source_y + 1; + source_y1 = clamped < 0 + ? 0u + : ((uint32_t)clamped >= source_height ? source_height - 1u + : (uint32_t)clamped); + } + lw_fixed_build_horizontal_row(source, source_stride, window_width, source_y0, + actual_width, workspace->x_offsets, + workspace->x_coefficients, workspace->row0); + lw_fixed_build_horizontal_row(source, source_stride, window_width, source_y1, + actual_width, workspace->x_offsets, + workspace->x_coefficients, workspace->row1); + for (ox = 0u; ox < actual_width; ++ox) { + uint32_t row_offset = ox * 3u; + for (channel = 0u; channel < 3u; ++channel) { + int32_t value = lw_fixed_blend(workspace->row0[row_offset + channel], + workspace->row1[row_offset + channel], beta0, + beta1); + size_t output_index = nhwc != 0 + ? ((size_t)destination + ox) * 3u + channel + : (size_t)channel * channel_plane + destination + ox; + output[output_index] = rec_lut[value]; + } + } + } + if (resized_width != NULL) { + *resized_width = actual_width; + } + return LW_STATUS_OK; +} + +lw_status lw_cls_preprocess_bgr_u8_fixed( + const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, float* output, + uint64_t output_element_count, uint32_t* resized_width, + lw_cls_preprocess_workspace* workspace) { + return cls_preprocess_bgr_u8_fixed_layout( + source, source_byte_count, source_width, source_height, source_stride, + output, output_element_count, resized_width, workspace, 0); +} + +lw_status lw_cls_preprocess_bgr_u8_fixed_nhwc( + const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, float* output, + uint64_t output_element_count, uint32_t* resized_width, + lw_cls_preprocess_workspace* workspace) { + return cls_preprocess_bgr_u8_fixed_layout( + source, source_byte_count, source_width, source_height, source_stride, + output, output_element_count, resized_width, workspace, 1); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop.c new file mode 100644 index 00000000..03ad1ae9 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop.c @@ -0,0 +1,255 @@ +#include "crop_internal.h" + +/* Perspective extraction and optional 180-degree rotation of detected lines. */ + +#include +#include +#include + +typedef struct crop_point { + double x; + double y; +} crop_point; + +static int load_points(const lw_detection_box* box, crop_point points[4]) { + const float values[8] = {box->x1, box->y1, box->x2, box->y2, + box->x3, box->y3, box->x4, box->y4}; + uint32_t index; + for (index = 0u; index < 4u; ++index) { + if (!isfinite(values[index * 2u]) || !isfinite(values[index * 2u + 1u])) + return 0; + points[index].x = values[index * 2u]; + points[index].y = values[index * 2u + 1u]; + } + return 1; +} + +static double point_distance(crop_point left, crop_point right) { + return hypot(right.x - left.x, right.y - left.y); +} + +lw_status lw_crop_quad_size(const lw_detection_box* box, uint32_t* crop_width, + uint32_t* crop_height, uint64_t* crop_byte_count) { + crop_point points[4]; + double width_value; + double height_value; + uint64_t width; + uint64_t height; + int rotate_vertical; + if (crop_width != NULL) + *crop_width = 0u; + if (crop_height != NULL) + *crop_height = 0u; + if (crop_byte_count != NULL) + *crop_byte_count = 0u; + if (box == NULL || crop_width == NULL || crop_height == NULL || crop_byte_count == NULL || + !load_points(box, points)) + return LW_STATUS_INVALID_ARGUMENT; + width_value = point_distance(points[0], points[1]); + height_value = point_distance(points[0], points[3]); + if (!isfinite(width_value) || !isfinite(height_value) || width_value < 1.0 || + height_value < 1.0 || width_value > UINT32_MAX || height_value > UINT32_MAX) + return LW_STATUS_INVALID_SHAPE; + width = (uint64_t)floor(width_value + 0.5); + height = (uint64_t)floor(height_value + 0.5); + if (width == 0u) + width = 1u; + if (height == 0u) + height = 1u; + rotate_vertical = (double)height >= (double)width * 1.5; + if (width > UINT64_MAX / height || width * height > UINT64_MAX / 3u) + return LW_STATUS_OUT_OF_BOUNDS; + *crop_byte_count = width * height * 3u; + *crop_width = (uint32_t)(rotate_vertical ? height : width); + *crop_height = (uint32_t)(rotate_vertical ? width : height); + return LW_STATUS_OK; +} + +static uint32_t clamp_coordinate(int64_t value, uint32_t limit) { + if (value < 0) + return 0u; + if ((uint64_t)value >= limit) + return limit - 1u; + return (uint32_t)value; +} + +static uint8_t sample_channel(const uint8_t* source, uint32_t source_width, uint32_t source_height, + uint32_t source_stride, double x, double y, uint32_t channel) { + int64_t x0_raw = (int64_t)floor(x); + int64_t y0_raw = (int64_t)floor(y); + uint32_t x0 = clamp_coordinate(x0_raw, source_width); + uint32_t x1 = clamp_coordinate(x0_raw + 1, source_width); + uint32_t y0 = clamp_coordinate(y0_raw, source_height); + uint32_t y1 = clamp_coordinate(y0_raw + 1, source_height); + double wx = x - x0_raw; + double wy = y - y0_raw; + double top_left = source[(size_t)((uint64_t)y0 * source_stride + (uint64_t)x0 * 3u + channel)]; + double top_right = source[(size_t)((uint64_t)y0 * source_stride + (uint64_t)x1 * 3u + channel)]; + double bottom_left = + source[(size_t)((uint64_t)y1 * source_stride + (uint64_t)x0 * 3u + channel)]; + double bottom_right = + source[(size_t)((uint64_t)y1 * source_stride + (uint64_t)x1 * 3u + channel)]; + double top = top_left + (top_right - top_left) * wx; + double bottom = bottom_left + (bottom_right - bottom_left) * wx; + double value = top + (bottom - top) * wy; + if (value <= 0.0) + return 0u; + if (value >= 255.0) + return 255u; + return (uint8_t)floor(value + 0.5); +} + +lw_status lw_crop_quad_bgr_u8_band(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, const lw_detection_box* box, + uint8_t* crop, uint64_t crop_capacity, + uint32_t* crop_width, uint32_t* crop_height, + uint64_t* crop_byte_count, uint32_t row_begin, + uint32_t row_end); + +uint32_t lw_crop_quad_unrotated_height(const lw_detection_box* box) { + crop_point points[4]; + if (box == NULL || !load_points(box, points)) return 0u; + return (uint32_t)floor(point_distance(points[0], points[3]) + 0.5); +} + +lw_status lw_crop_quad_bgr_u8_band(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, const lw_detection_box* box, + uint8_t* crop, uint64_t crop_capacity, + uint32_t* crop_width, uint32_t* crop_height, + uint64_t* crop_byte_count, uint32_t row_begin, + uint32_t row_end) { + crop_point points[4]; + uint32_t output_width; + uint32_t output_height; + uint64_t required_bytes; + uint32_t unrotated_width; + uint32_t unrotated_height; + int rotate_vertical; + double dx1; + double dx2; + double dx3; + double dy1; + double dy2; + double dy3; + double a; + double b; + double c; + double d; + double e; + double f; + double g = 0.0; + double h = 0.0; + uint64_t row_bytes; + uint64_t required_source_bytes; + uint32_t y; + lw_status status; + if (crop_width == NULL || crop_height == NULL || crop_byte_count == NULL) + return LW_STATUS_INVALID_ARGUMENT; + status = lw_crop_quad_size(box, &output_width, &output_height, &required_bytes); + if (status != LW_STATUS_OK) + return status; + if (source == NULL || crop == NULL || source_width == 0u || source_height == 0u || + !load_points(box, points)) + return LW_STATUS_INVALID_ARGUMENT; + row_bytes = (uint64_t)source_width * 3u; + if (row_bytes > UINT32_MAX || source_stride < row_bytes || + (uint64_t)(source_height - 1u) * source_stride > UINT64_MAX - row_bytes) + return LW_STATUS_INVALID_SHAPE; + required_source_bytes = (uint64_t)(source_height - 1u) * source_stride + row_bytes; + if (source_byte_count < required_source_bytes || required_source_bytes > SIZE_MAX) + return LW_STATUS_INVALID_SHAPE; + if (crop_capacity < required_bytes || required_bytes > SIZE_MAX) + return LW_STATUS_OUT_OF_BOUNDS; + unrotated_width = (uint32_t)floor(point_distance(points[0], points[1]) + 0.5); + unrotated_height = (uint32_t)floor(point_distance(points[0], points[3]) + 0.5); + if (row_begin >= row_end || row_end > unrotated_height) + return LW_STATUS_INVALID_ARGUMENT; + rotate_vertical = (double)unrotated_height >= (double)unrotated_width * 1.5; + + /* Solve the projective mapping from normalized crop coordinates (u, v) to + * the source quadrilateral. g/h are zero for an affine parallelogram. */ + dx1 = points[1].x - points[2].x; + dx2 = points[3].x - points[2].x; + dx3 = points[0].x - points[1].x + points[2].x - points[3].x; + dy1 = points[1].y - points[2].y; + dy2 = points[3].y - points[2].y; + dy3 = points[0].y - points[1].y + points[2].y - points[3].y; + if (fabs(dx3) > 1.0e-12 || fabs(dy3) > 1.0e-12) { + double denominator = dx1 * dy2 - dx2 * dy1; + if (!isfinite(denominator) || fabs(denominator) <= 1.0e-12) + return LW_STATUS_INVALID_SHAPE; + g = (dx3 * dy2 - dx2 * dy3) / denominator; + h = (dx1 * dy3 - dx3 * dy1) / denominator; + } + a = points[1].x - points[0].x + g * points[1].x; + b = points[3].x - points[0].x + h * points[3].x; + c = points[0].x; + d = points[1].y - points[0].y + g * points[1].y; + e = points[3].y - points[0].y + h * points[3].y; + f = points[0].y; + /* Sample backwards from destination to source. Backward mapping avoids + * holes, while bilinear sampling gives smoother character edges. Bands + * are row-disjoint output ranges with identical per-pixel math, so the + * parallel phase writes bit-identical crops. */ + for (y = row_begin; y < row_end; ++y) { + uint32_t x; + double v = (double)y / unrotated_height; + for (x = 0u; x < unrotated_width; ++x) { + double u = (double)x / unrotated_width; + double denominator = g * u + h * v + 1.0; + double source_x; + double source_y; + /* Very tall regions are rotated into the horizontal orientation + * expected by PP-OCR REC before optional CLS correction. */ + uint32_t destination_x = rotate_vertical ? unrotated_height - 1u - y : x; + uint32_t destination_y = rotate_vertical ? x : y; + uint64_t destination = ((uint64_t)destination_y * output_width + destination_x) * 3u; + uint32_t channel; + if (!isfinite(denominator) || fabs(denominator) <= 1.0e-12) + return LW_STATUS_INVALID_SHAPE; + source_x = (a * u + b * v + c) / denominator; + source_y = (d * u + e * v + f) / denominator; + if (!isfinite(source_x) || !isfinite(source_y)) + return LW_STATUS_INVALID_SHAPE; + for (channel = 0u; channel < 3u; ++channel) { + crop[(size_t)(destination + channel)] = + sample_channel(source, source_width, source_height, source_stride, source_x, + source_y, channel); + } + } + } + *crop_width = output_width; + *crop_height = output_height; + *crop_byte_count = required_bytes; + return LW_STATUS_OK; +} + +lw_status lw_crop_quad_bgr_u8(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, uint32_t source_stride, + const lw_detection_box* box, uint8_t* crop, uint64_t crop_capacity, + uint32_t* crop_width, uint32_t* crop_height, + uint64_t* crop_byte_count) { + uint32_t unrotated_height = lw_crop_quad_unrotated_height(box); + return lw_crop_quad_bgr_u8_band(source, source_byte_count, source_width, source_height, + source_stride, box, crop, crop_capacity, crop_width, + crop_height, crop_byte_count, 0u, unrotated_height); +} + +void lw_rotate_bgr_u8_180(uint8_t* pixels, uint32_t width, uint32_t height) { + uint64_t pixel_count; + uint64_t left; + if (pixels == NULL) + return; + pixel_count = (uint64_t)width * height; + for (left = 0u; left < pixel_count / 2u; ++left) { + uint64_t right = pixel_count - 1u - left; + uint32_t channel; + for (channel = 0u; channel < 3u; ++channel) { + uint8_t value = pixels[(size_t)(left * 3u + channel)]; + pixels[(size_t)(left * 3u + channel)] = pixels[(size_t)(right * 3u + channel)]; + pixels[(size_t)(right * 3u + channel)] = value; + } + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop_buffer.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop_buffer.c new file mode 100644 index 00000000..08731cc7 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop_buffer.c @@ -0,0 +1,91 @@ +#include "crop_buffer_internal.h" + +#include "error_internal.h" + +#include +#include +#include +#include + +#define LW_CROP_BUFFER_MIN_CAPACITY UINT64_C(4096) + +static int crop_size_fits(uint64_t bytes) { +#if SIZE_MAX < UINT64_MAX + return bytes <= (uint64_t)SIZE_MAX; +#else + (void)bytes; + return 1; +#endif +} + +static uint64_t next_capacity(uint64_t current, uint64_t required) { + uint64_t value = current < LW_CROP_BUFFER_MIN_CAPACITY + ? LW_CROP_BUFFER_MIN_CAPACITY + : current; + while (value < required) { + if (value > UINT64_MAX / 2u) { + return required; + } + value *= 2u; + } + return value; +} + +void lw_crop_buffer_init(lw_crop_buffer* buffer, uint64_t retained_limit) { + if (buffer == NULL) { + return; + } + memset(buffer, 0, sizeof(*buffer)); + buffer->retained_limit = retained_limit; +} + +void lw_crop_buffer_free(lw_crop_buffer* buffer) { + if (buffer == NULL) { + return; + } + free(buffer->data); + memset(buffer, 0, sizeof(*buffer)); +} + +lw_status lw_crop_buffer_reserve(lw_crop_buffer* buffer, uint64_t required_bytes, + lw_error* error) { + uint64_t capacity; + uint8_t* resized; + + if (buffer == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "crop buffer is required"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (required_bytes <= buffer->capacity) { + return LW_STATUS_OK; + } + if (!crop_size_fits(required_bytes)) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, + "crop buffer exceeds the platform address space"); + return LW_STATUS_OUT_OF_BOUNDS; + } + + capacity = next_capacity(buffer->capacity, required_bytes); + if (!crop_size_fits(capacity)) { + capacity = required_bytes; + } + resized = (uint8_t*)realloc(buffer->data, (size_t)capacity); + if (resized == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, + "unable to grow OCR worker crop buffer"); + return LW_STATUS_OUT_OF_MEMORY; + } + buffer->data = resized; + buffer->capacity = capacity; + return LW_STATUS_OK; +} + +void lw_crop_buffer_trim_after_request(lw_crop_buffer* buffer) { + if (buffer == NULL || buffer->retained_limit == 0u || + buffer->capacity <= buffer->retained_limit) { + return; + } + free(buffer->data); + buffer->data = NULL; + buffer->capacity = 0u; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop_buffer_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop_buffer_internal.h new file mode 100644 index 00000000..576c3acc --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop_buffer_internal.h @@ -0,0 +1,20 @@ +#ifndef LW_CROP_BUFFER_INTERNAL_H +#define LW_CROP_BUFFER_INTERNAL_H + +#include "lw_infer.h" + +#include + +typedef struct lw_crop_buffer { + uint8_t* data; + uint64_t capacity; + uint64_t retained_limit; +} lw_crop_buffer; + +void lw_crop_buffer_init(lw_crop_buffer* buffer, uint64_t retained_limit); +void lw_crop_buffer_free(lw_crop_buffer* buffer); +lw_status lw_crop_buffer_reserve(lw_crop_buffer* buffer, uint64_t required_bytes, + lw_error* error); +void lw_crop_buffer_trim_after_request(lw_crop_buffer* buffer); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop_internal.h new file mode 100644 index 00000000..fc0ccb7a --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/crop_internal.h @@ -0,0 +1,33 @@ +#ifndef LW_CROP_INTERNAL_H +#define LW_CROP_INTERNAL_H + +/* Internal quadrilateral crop helpers shared by the composed OCR pipeline. */ + +#include "lw_infer.h" + +#include + +lw_status lw_crop_quad_size(const lw_detection_box* box, uint32_t* crop_width, + uint32_t* crop_height, uint64_t* crop_byte_count); + +lw_status lw_crop_quad_bgr_u8(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, uint32_t source_stride, + const lw_detection_box* box, uint8_t* crop, uint64_t crop_capacity, + uint32_t* crop_width, uint32_t* crop_height, + uint64_t* crop_byte_count); + +/* Row-band variant for the parallel crop phase: identical per-pixel math, + * restricted to [row_begin, row_end) of the unrotated source rows. */ +lw_status lw_crop_quad_bgr_u8_band(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, const lw_detection_box* box, + uint8_t* crop, uint64_t crop_capacity, + uint32_t* crop_width, uint32_t* crop_height, + uint64_t* crop_byte_count, uint32_t row_begin, + uint32_t row_end); + +uint32_t lw_crop_quad_unrotated_height(const lw_detection_box* box); + +void lw_rotate_bgr_u8_180(uint8_t* pixels, uint32_t width, uint32_t height); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/ctc_decode.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/ctc_decode.c new file mode 100644 index 00000000..303cf46e --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/ctc_decode.c @@ -0,0 +1,611 @@ +#include "rec_internal.h" + +/* UTF-8 dictionary loading and greedy CTC collapse for REC probabilities. */ + +#include "error_internal.h" + +#include +#include +#include +#include + +#if defined(_WIN32) +# define WIN32_LEAN_AND_MEAN +# include +#endif + +#define LW_REC_MAX_DICTIONARY_SIZE (UINT64_C(1024) * UINT64_C(1024)) + +struct lw_rec_dictionary { + uint32_t ref_count; + uint8_t* bytes; + uint32_t byte_count; + uint32_t entry_count; + uint32_t* offsets; + uint32_t* lengths; +}; + +#if defined(_WIN32) +static FILE* open_read_utf8(const char* path_utf8) { + int wide_count; + wchar_t* wide_path; + FILE* file = NULL; + wide_count = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path_utf8, -1, NULL, 0); + if (wide_count <= 0) { + return NULL; + } + wide_path = (wchar_t*)malloc((size_t)wide_count * sizeof(*wide_path)); + if (wide_path == NULL) { + return NULL; + } + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path_utf8, -1, wide_path, wide_count) > + 0 && + _wfopen_s(&file, wide_path, L"rb") != 0) { + file = NULL; + } + free(wide_path); + return file; +} +#else +static FILE* open_read_utf8(const char* path_utf8) { + return fopen(path_utf8, "rb"); +} +#endif + +static int utf8_valid(const uint8_t* bytes, uint32_t count) { + uint32_t index = 0u; + while (index < count) { + uint8_t first = bytes[index++]; + uint32_t continuation_count; + uint32_t codepoint; + uint32_t minimum; + uint32_t continuation; + if (first == 0u) { + return 0; + } + if (first < 0x80u) { + continue; + } + if (first >= 0xc2u && first <= 0xdfu) { + continuation_count = 1u; + codepoint = first & 0x1fu; + minimum = 0x80u; + } else if (first >= 0xe0u && first <= 0xefu) { + continuation_count = 2u; + codepoint = first & 0x0fu; + minimum = 0x800u; + } else if (first >= 0xf0u && first <= 0xf4u) { + continuation_count = 3u; + codepoint = first & 0x07u; + minimum = 0x10000u; + } else { + return 0; + } + if (continuation_count > count - index) { + return 0; + } + for (continuation = 0u; continuation < continuation_count; ++continuation) { + uint8_t value = bytes[index++]; + if ((value & 0xc0u) != 0x80u) { + return 0; + } + codepoint = (codepoint << 6) | (value & 0x3fu); + } + if (codepoint < minimum || codepoint > 0x10ffffu || + (codepoint >= 0xd800u && codepoint <= 0xdfffu)) { + return 0; + } + } + return 1; +} + +static int count_entries(const uint8_t* bytes, uint32_t count, uint32_t start, uint32_t* result) { + uint32_t entries = 0u; + uint32_t cursor = start; + int has_non_empty_label = 0; + while (cursor < count) { + uint32_t end = cursor; + uint32_t label_end; + while (end < count && bytes[end] != (uint8_t)'\n') { + ++end; + } + label_end = end; + if (label_end > cursor && bytes[label_end - 1u] == (uint8_t)'\r') { + --label_end; + } + if (!utf8_valid(bytes + cursor, label_end - cursor) || entries == UINT32_MAX - 2u) { + return 0; + } + if (label_end != cursor) { + has_non_empty_label = 1; + } + ++entries; + cursor = end < count ? end + 1u : count; + } + if (entries == 0u || !has_non_empty_label) { + return 0; + } + *result = entries; + return 1; +} + +lw_status lw_rec_dictionary_load(const char* path_utf8, lw_rec_dictionary** out_dictionary, + lw_error* error) { + FILE* file; + long length; + lw_rec_dictionary* dictionary; + size_t read_count; + uint32_t start; + uint32_t cursor; + uint32_t entry_index = 0u; + if (out_dictionary != NULL) { + *out_dictionary = NULL; + } + if (path_utf8 == NULL || path_utf8[0] == '\0' || out_dictionary == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "dictionary path and output handle are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + file = open_read_utf8(path_utf8); + if (file == NULL) { + lw_set_error(error, LW_STATUS_IO_ERROR, "unable to open recognition dictionary"); + return LW_STATUS_IO_ERROR; + } + if (fseek(file, 0, SEEK_END) != 0 || (length = ftell(file)) < 0 || + fseek(file, 0, SEEK_SET) != 0 || (uint64_t)length > LW_REC_MAX_DICTIONARY_SIZE) { + fclose(file); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "recognition dictionary size is invalid"); + return LW_STATUS_OUT_OF_BOUNDS; + } + dictionary = (lw_rec_dictionary*)calloc(1u, sizeof(*dictionary)); + if (dictionary == NULL) { + fclose(file); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate dictionary handle"); + return LW_STATUS_OUT_OF_MEMORY; + } + dictionary->ref_count = 1u; + dictionary->byte_count = (uint32_t)length; + dictionary->bytes = + (uint8_t*)malloc(dictionary->byte_count == 0u ? 1u : dictionary->byte_count); + if (dictionary->bytes == NULL) { + fclose(file); + lw_rec_dictionary_free(dictionary); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate dictionary bytes"); + return LW_STATUS_OUT_OF_MEMORY; + } + read_count = fread(dictionary->bytes, 1u, dictionary->byte_count, file); + fclose(file); + if (read_count != dictionary->byte_count) { + lw_rec_dictionary_free(dictionary); + lw_set_error(error, LW_STATUS_IO_ERROR, "unable to read complete recognition dictionary"); + return LW_STATUS_IO_ERROR; + } + start = dictionary->byte_count >= 3u && dictionary->bytes[0] == 0xefu && + dictionary->bytes[1] == 0xbbu && dictionary->bytes[2] == 0xbfu + ? 3u + : 0u; + if (!count_entries(dictionary->bytes, dictionary->byte_count, start, + &dictionary->entry_count)) { + lw_rec_dictionary_free(dictionary); + lw_set_error(error, LW_STATUS_INVALID_FORMAT, + "recognition dictionary is empty or contains invalid UTF-8"); + return LW_STATUS_INVALID_FORMAT; + } + dictionary->offsets = + (uint32_t*)malloc((size_t)dictionary->entry_count * sizeof(*dictionary->offsets)); + dictionary->lengths = + (uint32_t*)malloc((size_t)dictionary->entry_count * sizeof(*dictionary->lengths)); + if (dictionary->offsets == NULL || dictionary->lengths == NULL) { + lw_rec_dictionary_free(dictionary); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate dictionary index"); + return LW_STATUS_OUT_OF_MEMORY; + } + cursor = start; + while (cursor < dictionary->byte_count) { + uint32_t end = cursor; + uint32_t label_end; + while (end < dictionary->byte_count && dictionary->bytes[end] != (uint8_t)'\n') { + ++end; + } + label_end = end; + if (label_end > cursor && dictionary->bytes[label_end - 1u] == (uint8_t)'\r') { + --label_end; + } + dictionary->offsets[entry_index] = cursor; + dictionary->lengths[entry_index] = label_end - cursor; + ++entry_index; + cursor = end < dictionary->byte_count ? end + 1u : dictionary->byte_count; + } + *out_dictionary = dictionary; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +void lw_rec_dictionary_free(lw_rec_dictionary* dictionary) { + if (dictionary == NULL) { + return; + } + if (dictionary->ref_count > 1u) { + --dictionary->ref_count; + return; + } + free(dictionary->lengths); + free(dictionary->offsets); + free(dictionary->bytes); + free(dictionary); +} + +void lw_rec_dictionary_retain(lw_rec_dictionary* dictionary) { + if (dictionary != NULL && dictionary->ref_count != UINT32_MAX) { + ++dictionary->ref_count; + } +} + +uint32_t lw_rec_dictionary_class_count(const lw_rec_dictionary* dictionary) { + return dictionary == NULL ? 0u : dictionary->entry_count + 2u; +} + +uint32_t lw_rec_dictionary_max_label_byte_count(const lw_rec_dictionary* dictionary) { + uint32_t maximum = 1u; + uint32_t index; + if (dictionary == NULL) { + return 0u; + } + for (index = 0u; index < dictionary->entry_count; ++index) { + if (dictionary->lengths[index] > maximum) { + maximum = dictionary->lengths[index]; + } + } + return maximum; +} + +static lw_status decode_pass(const lw_rec_dictionary* dictionary, const float* probabilities, + uint32_t time_steps, uint32_t class_count, char* text_utf8, + uint64_t text_capacity, uint64_t* text_bytes, float* score, + uint32_t* emitted_count) { + uint64_t bytes = 0u; + double score_sum = 0.0; + uint32_t emitted = 0u; + uint32_t previous = 0u; + uint32_t step; + /* Greedy CTC decoding chooses the best class at each time step. Class 0 is + * blank; consecutive repeats collapse unless a blank separates them. */ + for (step = 0u; step < time_steps; ++step) { + const float* row = probabilities + (size_t)((uint64_t)step * class_count); + uint32_t best_index = 0u; + float best_value = row[0]; + uint32_t class_index; + if (!isfinite(best_value)) { + return LW_STATUS_INVALID_ARGUMENT; + } + for (class_index = 1u; class_index < class_count; ++class_index) { + float value = row[class_index]; + if (!isfinite(value)) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (value > best_value) { + best_value = value; + best_index = class_index; + } + } + if (best_index != 0u && (step == 0u || best_index != previous)) { + const uint8_t* label; + uint32_t label_length; + if (best_index == dictionary->entry_count + 1u) { + static const uint8_t space = (uint8_t)' '; + label = &space; + label_length = 1u; + } else { + uint32_t dictionary_index = best_index - 1u; + label = dictionary->bytes + dictionary->offsets[dictionary_index]; + label_length = dictionary->lengths[dictionary_index]; + } + if (bytes > UINT64_MAX - label_length) { + return LW_STATUS_OUT_OF_BOUNDS; + } + if (text_utf8 != NULL) { + /* Reserve one byte for the trailing NUL before touching the + * caller buffer. This also keeps the known-capacity fast path + * safe if its internal capacity guarantee is ever violated. */ + if (bytes >= text_capacity || label_length >= text_capacity - bytes) { + return LW_STATUS_OUT_OF_BOUNDS; + } + memcpy(text_utf8 + (size_t)bytes, label, label_length); + } + bytes += label_length; + score_sum += best_value; + ++emitted; + } + previous = best_index; + } + if (text_utf8 != NULL) { + text_utf8[(size_t)bytes] = '\0'; + } + *text_bytes = bytes; + *score = emitted == 0u ? 0.0f : (float)(score_sum / emitted); + *emitted_count = emitted; + return LW_STATUS_OK; +} + +static lw_status decode_greedy_pass(const lw_rec_dictionary* dictionary, + const uint32_t* best_indices, + const float* best_probabilities, uint32_t time_steps, + uint32_t class_count, char* text_utf8, + uint64_t text_capacity, uint64_t* text_bytes, float* score, + uint32_t* emitted_count) { + uint64_t bytes = 0u; + double score_sum = 0.0; + uint32_t emitted = 0u; + uint32_t previous = 0u; + uint32_t step; + for (step = 0u; step < time_steps; ++step) { + uint32_t best_index = best_indices[step]; + float best_value = best_probabilities[step]; + if (best_index >= class_count || !isfinite(best_value)) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (best_index != 0u && (step == 0u || best_index != previous)) { + const uint8_t* label; + uint32_t label_length; + if (best_index == dictionary->entry_count + 1u) { + static const uint8_t space = (uint8_t)' '; + label = &space; + label_length = 1u; + } else { + uint32_t dictionary_index = best_index - 1u; + label = dictionary->bytes + dictionary->offsets[dictionary_index]; + label_length = dictionary->lengths[dictionary_index]; + } + if (bytes > UINT64_MAX - label_length) { + return LW_STATUS_OUT_OF_BOUNDS; + } + if (text_utf8 != NULL) { + if (bytes >= text_capacity || label_length >= text_capacity - bytes) { + return LW_STATUS_OUT_OF_BOUNDS; + } + memcpy(text_utf8 + (size_t)bytes, label, label_length); + } + bytes += label_length; + score_sum += best_value; + ++emitted; + } + previous = best_index; + } + if (text_utf8 != NULL) { + text_utf8[(size_t)bytes] = '\0'; + } + *text_bytes = bytes; + *score = emitted == 0u ? 0.0f : (float)(score_sum / emitted); + *emitted_count = emitted; + return LW_STATUS_OK; +} + +lw_status lw_rec_ctc_decode_f32(const lw_rec_dictionary* dictionary, const float* probabilities, + uint64_t probability_element_count, uint32_t time_steps, + uint32_t class_count, char* text_utf8, uint64_t text_capacity, + uint64_t* required_capacity, float* score, uint32_t* emitted_count, + lw_error* error) { + uint64_t expected_elements; + uint64_t text_bytes; + float decoded_score; + uint32_t decoded_count; + lw_status status; + if (required_capacity != NULL) { + *required_capacity = 0u; + } + if (score != NULL) { + *score = 0.0f; + } + if (emitted_count != NULL) { + *emitted_count = 0u; + } + if (dictionary == NULL || probabilities == NULL || required_capacity == NULL || score == NULL || + emitted_count == NULL || time_steps == 0u || class_count == 0u || + (text_utf8 == NULL && text_capacity != 0u)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "dictionary, probabilities, and decode outputs are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + expected_elements = (uint64_t)time_steps * class_count; + if (probability_element_count != expected_elements || + class_count != dictionary->entry_count + 2u || + expected_elements > SIZE_MAX / sizeof(float)) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "CTC probability shape or dictionary class count is invalid"); + return LW_STATUS_INVALID_SHAPE; + } + /* First pass computes exact UTF-8 capacity without touching caller memory. */ + status = decode_pass(dictionary, probabilities, time_steps, class_count, NULL, 0u, &text_bytes, + &decoded_score, &decoded_count); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "CTC probabilities contain invalid values"); + return status; + } + if (text_bytes >= SIZE_MAX) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "CTC text capacity overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + *required_capacity = text_bytes + 1u; + *score = decoded_score; + *emitted_count = decoded_count; + if (text_utf8 == NULL) { + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + } + if (text_capacity < *required_capacity) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "CTC text buffer is too small"); + return LW_STATUS_OUT_OF_BOUNDS; + } + status = decode_pass(dictionary, probabilities, time_steps, class_count, text_utf8, + text_capacity, &text_bytes, &decoded_score, &decoded_count); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "CTC decoding failed"); + return status; + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +lw_status lw_rec_ctc_decode_known_capacity_f32(const lw_rec_dictionary* dictionary, + const float* probabilities, + uint64_t probability_element_count, + uint32_t time_steps, uint32_t class_count, + char* text_utf8, uint64_t text_capacity, + uint64_t* required_capacity, float* score, + uint32_t* emitted_count, lw_error* error) { + uint64_t expected_elements; + uint64_t text_bytes = 0u; + float decoded_score = 0.0f; + uint32_t decoded_count = 0u; + lw_status status; + if (required_capacity != NULL) { + *required_capacity = 0u; + } + if (score != NULL) { + *score = 0.0f; + } + if (emitted_count != NULL) { + *emitted_count = 0u; + } + if (dictionary == NULL || probabilities == NULL || text_utf8 == NULL || text_capacity == 0u || + required_capacity == NULL || score == NULL || emitted_count == NULL || time_steps == 0u || + class_count == 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "known-capacity CTC decode inputs are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + expected_elements = (uint64_t)time_steps * class_count; + if (probability_element_count != expected_elements || + class_count != dictionary->entry_count + 2u || + expected_elements > SIZE_MAX / sizeof(float)) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "CTC probability shape or dictionary class count is invalid"); + return LW_STATUS_INVALID_SHAPE; + } + status = decode_pass(dictionary, probabilities, time_steps, class_count, text_utf8, + text_capacity, &text_bytes, &decoded_score, &decoded_count); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, + status == LW_STATUS_OUT_OF_BOUNDS + ? "CTC text buffer is too small" + : "CTC probabilities contain invalid values"); + return status; + } + if (text_bytes >= SIZE_MAX) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "CTC text capacity overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + *required_capacity = text_bytes + 1u; + *score = decoded_score; + *emitted_count = decoded_count; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + + +lw_status lw_rec_ctc_decode_greedy_f32(const lw_rec_dictionary* dictionary, + const uint32_t* best_indices, + const float* best_probabilities, uint32_t time_steps, + uint32_t class_count, char* text_utf8, + uint64_t text_capacity, uint64_t* required_capacity, + float* score, uint32_t* emitted_count, lw_error* error) { + uint64_t text_bytes; + float decoded_score; + uint32_t decoded_count; + lw_status status; + if (required_capacity != NULL) { + *required_capacity = 0u; + } + if (score != NULL) { + *score = 0.0f; + } + if (emitted_count != NULL) { + *emitted_count = 0u; + } + if (dictionary == NULL || best_indices == NULL || best_probabilities == NULL || + required_capacity == NULL || score == NULL || emitted_count == NULL || + time_steps == 0u || class_count != dictionary->entry_count + 2u || + (text_utf8 == NULL && text_capacity != 0u)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "dictionary, greedy classes, and decode outputs are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + status = decode_greedy_pass(dictionary, best_indices, best_probabilities, time_steps, + class_count, NULL, 0u, &text_bytes, &decoded_score, + &decoded_count); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "CTC greedy outputs contain invalid values"); + return status; + } + if (text_bytes >= SIZE_MAX) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "CTC text capacity overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + *required_capacity = text_bytes + 1u; + *score = decoded_score; + *emitted_count = decoded_count; + if (text_utf8 == NULL) { + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + } + if (text_capacity < *required_capacity) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "CTC text buffer is too small"); + return LW_STATUS_OUT_OF_BOUNDS; + } + status = decode_greedy_pass(dictionary, best_indices, best_probabilities, time_steps, + class_count, text_utf8, text_capacity, &text_bytes, + &decoded_score, &decoded_count); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "CTC greedy decoding failed"); + return status; + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +lw_status lw_rec_ctc_decode_greedy_known_capacity_f32( + const lw_rec_dictionary* dictionary, const uint32_t* best_indices, + const float* best_probabilities, uint32_t time_steps, uint32_t class_count, + char* text_utf8, uint64_t text_capacity, uint64_t* required_capacity, float* score, + uint32_t* emitted_count, lw_error* error) { + uint64_t text_bytes = 0u; + float decoded_score = 0.0f; + uint32_t decoded_count = 0u; + lw_status status; + if (required_capacity != NULL) { + *required_capacity = 0u; + } + if (score != NULL) { + *score = 0.0f; + } + if (emitted_count != NULL) { + *emitted_count = 0u; + } + if (dictionary == NULL || best_indices == NULL || best_probabilities == NULL || + text_utf8 == NULL || text_capacity == 0u || required_capacity == NULL || + score == NULL || emitted_count == NULL || time_steps == 0u || + class_count != dictionary->entry_count + 2u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "known-capacity CTC greedy decode inputs are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + status = decode_greedy_pass(dictionary, best_indices, best_probabilities, time_steps, + class_count, text_utf8, text_capacity, &text_bytes, + &decoded_score, &decoded_count); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, + status == LW_STATUS_OUT_OF_BOUNDS + ? "CTC text buffer is too small" + : "CTC greedy outputs contain invalid values"); + return status; + } + if (text_bytes >= SIZE_MAX) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "CTC text capacity overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + *required_capacity = text_bytes + 1u; + *score = decoded_score; + *emitted_count = decoded_count; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/db_postprocess.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/db_postprocess.c new file mode 100644 index 00000000..cfd472bb --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/db_postprocess.c @@ -0,0 +1,1117 @@ +#include "det_internal.h" +#include "profile_internal.h" +#include "../simd/simd_kernels.h" + +/* + * Bounded DB-style postprocessing: threshold the probability map, collect + * connected components, fit/expand quadrilaterals and restore source-image + * coordinates. Candidate limits prevent hostile images from growing memory. + */ + +#include +#include +#include +#include +#include +#include + +typedef struct det_point { + float x; + float y; +} det_point; + +/* Keep allocation overflow checks meaningful on both 32-bit and 64-bit + * targets. Passing the count through uint64_t avoids comparisons that GCC can + * prove are always false when the original count is only uint32_t. */ +static int allocation_fits(uint64_t count, size_t element_size) { + return element_size != 0u && count <= (uint64_t)(SIZE_MAX / element_size); +} + +typedef struct det_rectangle { + float ux; + float uy; + float vx; + float vy; + float min_u; + float max_u; + float min_v; + float max_v; +} det_rectangle; + +static int compare_points(const void* left_value, const void* right_value) { + const det_point* left = (const det_point*)left_value; + const det_point* right = (const det_point*)right_value; + if (left->x < right->x) + return -1; + if (left->x > right->x) + return 1; + if (left->y < right->y) + return -1; + if (left->y > right->y) + return 1; + return 0; +} + +static float cross(det_point origin, det_point first, det_point second) { + return (first.x - origin.x) * (second.y - origin.y) - + (first.y - origin.y) * (second.x - origin.x); +} + +static uint32_t convex_hull(det_point* points, uint32_t point_count, det_point* hull) { + uint32_t unique_count = 0u; + uint32_t index; + uint32_t count = 0u; + if (point_count < 3u) { + return 0u; + } + qsort(points, point_count, sizeof(*points), compare_points); + for (index = 0u; index < point_count; ++index) { + if (unique_count == 0u || points[index].x != points[unique_count - 1u].x || + points[index].y != points[unique_count - 1u].y) { + points[unique_count++] = points[index]; + } + } + if (unique_count < 3u) { + return 0u; + } + for (index = 0u; index < unique_count; ++index) { + while (count >= 2u && cross(hull[count - 2u], hull[count - 1u], points[index]) <= 0.0f) { + --count; + } + hull[count++] = points[index]; + } + { + uint32_t lower_count = count; + for (index = unique_count - 1u; index > 0u; --index) { + while (count > lower_count && + cross(hull[count - 2u], hull[count - 1u], points[index - 1u]) <= 0.0f) { + --count; + } + hull[count++] = points[index - 1u]; + } + } + return count > 1u ? count - 1u : 0u; +} + +static int minimum_rectangle(const det_point* hull, uint32_t hull_count, det_rectangle* rectangle) { + float best_area = INFINITY; + uint32_t edge; + if (hull == NULL || rectangle == NULL || hull_count < 3u) { + return 0; + } + for (edge = 0u; edge < hull_count; ++edge) { + det_point current = hull[edge]; + det_point next = hull[(edge + 1u) % hull_count]; + float dx = next.x - current.x; + float dy = next.y - current.y; + float length = sqrtf(dx * dx + dy * dy); + float ux; + float uy; + float vx; + float vy; + float min_u = INFINITY; + float max_u = -INFINITY; + float min_v = INFINITY; + float max_v = -INFINITY; + uint32_t point; + float area; + if (length <= 0.0f) { + continue; + } + ux = dx / length; + uy = dy / length; + vx = -uy; + vy = ux; + for (point = 0u; point < hull_count; ++point) { + float projection_u = hull[point].x * ux + hull[point].y * uy; + float projection_v = hull[point].x * vx + hull[point].y * vy; + if (projection_u < min_u) + min_u = projection_u; + if (projection_u > max_u) + max_u = projection_u; + if (projection_v < min_v) + min_v = projection_v; + if (projection_v > max_v) + max_v = projection_v; + } + area = (max_u - min_u) * (max_v - min_v); + if (area < best_area) { + best_area = area; + rectangle->ux = ux; + rectangle->uy = uy; + rectangle->vx = vx; + rectangle->vy = vy; + rectangle->min_u = min_u; + rectangle->max_u = max_u; + rectangle->min_v = min_v; + rectangle->max_v = max_v; + } + } + return isfinite(best_area) && best_area > 0.0f; +} + +static det_point from_projection(const det_rectangle* rectangle, float projection_u, + float projection_v) { + det_point point; + point.x = projection_u * rectangle->ux + projection_v * rectangle->vx; + point.y = projection_u * rectangle->uy + projection_v * rectangle->vy; + return point; +} + +static void rectangle_points(const det_rectangle* rectangle, float expansion, det_point points[4]) { + points[0] = + from_projection(rectangle, rectangle->min_u - expansion, rectangle->min_v - expansion); + points[1] = + from_projection(rectangle, rectangle->max_u + expansion, rectangle->min_v - expansion); + points[2] = + from_projection(rectangle, rectangle->max_u + expansion, rectangle->max_v + expansion); + points[3] = + from_projection(rectangle, rectangle->min_u - expansion, rectangle->max_v + expansion); +} + +static float cross_points(det_point origin, det_point first, det_point second) { + return (first.x - origin.x) * (second.y - origin.y) - + (first.y - origin.y) * (second.x - origin.x); +} + +/* PaddleOCR order_points_clockwise: x+y minimal is top-left, maximal is + * bottom-right, and the y-x difference splits the remaining two corners. A + * 45-degree tie falls back to centroid-angle ordering. */ +static int is_positive_convex_cycle(const det_point points[4]) { + uint32_t i; + for (i = 0u; i < 4u; ++i) { + if (cross_points(points[i], points[(i + 1u) & 3u], points[(i + 2u) & 3u]) <= 0.0f) { + return 0; + } + } + return 1; +} + +static void order_by_centroid_angle(det_point points[4], const det_point input[4]) { + float center_x = 0.0f; + float center_y = 0.0f; + float angles[4]; + uint32_t order[4] = {0u, 1u, 2u, 3u}; + uint32_t start = 0u; + uint32_t i; + for (i = 0u; i < 4u; ++i) { + center_x += input[i].x; + center_y += input[i].y; + } + center_x /= 4.0f; + center_y /= 4.0f; + for (i = 0u; i < 4u; ++i) { + angles[i] = atan2f(input[i].y - center_y, input[i].x - center_x); + } + for (i = 1u; i < 4u; ++i) { + int32_t j = (int32_t)i - 1; + uint32_t index = order[i]; + while (j >= 0 && angles[order[j]] > angles[index]) { + order[(uint32_t)j + 1u] = order[(uint32_t)j]; + --j; + } + order[(uint32_t)j + 1u] = index; + } + for (i = 1u; i < 4u; ++i) { + det_point candidate = input[order[i]]; + det_point best = input[order[start]]; + float candidate_sum = candidate.x + candidate.y; + float best_sum = best.x + best.y; + if (candidate_sum < best_sum || + (candidate_sum == best_sum && + (candidate.y < best.y || (candidate.y == best.y && candidate.x < best.x)))) { + start = i; + } + } + for (i = 0u; i < 4u; ++i) { + points[i] = input[order[(start + i) & 3u]]; + } +} + +static void order_clockwise(det_point points[4]) { + det_point input[4]; + uint32_t min_sum_index = 0u; + uint32_t max_sum_index = 0u; + float min_sum; + float max_sum; + uint32_t first_remaining = UINT32_MAX; + uint32_t second_remaining = UINT32_MAX; + det_point first; + det_point second; + float first_diff; + float second_diff; + det_point top_right; + det_point bottom_left; + uint32_t i; + memcpy(input, points, sizeof(input)); + min_sum = max_sum = input[0].x + input[0].y; + for (i = 1u; i < 4u; ++i) { + float sum = input[i].x + input[i].y; + if (sum < min_sum) { + min_sum = sum; + min_sum_index = i; + } + if (sum > max_sum) { + max_sum = sum; + max_sum_index = i; + } + } + for (i = 0u; i < 4u; ++i) { + if (i == min_sum_index || i == max_sum_index) continue; + if (first_remaining == UINT32_MAX) { + first_remaining = i; + } else { + second_remaining = i; + } + } + first = input[first_remaining]; + second = input[second_remaining]; + first_diff = first.y - first.x; + second_diff = second.y - second.x; + top_right = first_diff < second_diff ? first : second; + bottom_left = first_diff < second_diff ? second : first; + points[0] = input[min_sum_index]; + points[1] = top_right; + points[2] = input[max_sum_index]; + points[3] = bottom_left; + if (!is_positive_convex_cycle(points)) { + order_by_centroid_angle(points, input); + } +} + +static double signed_area(const det_point* points, uint32_t count) { + double area = 0.0; + uint32_t i; + for (i = 0u; i < count; ++i) { + det_point a = points[i]; + det_point b = points[(i + 1u) % count]; + area += (double)a.x * b.y - (double)b.x * a.y; + } + return area * 0.5; +} + +static float round_half_away(double value) { + return (float)(value < 0.0 ? (int64_t)(value - 0.5) : (int64_t)(value + 0.5)); +} + +static int64_t round_half_even_i64(double value) { + double rounded = floor(value + 0.5); + if (rounded - value == 0.5 && ((int64_t)rounded & 1) != 0) { + rounded -= 1.0; + } + return (int64_t)rounded; +} + +static float round_half_even_f(float value) { + float rounded = floorf(value + 0.5f); + if (rounded - value == 0.5f && ((int64_t)rounded & 1) != 0) { + rounded -= 1.0f; + } + return rounded; +} + +static float clamp_coord(float value, float limit) { + if (value < 0.0f) return 0.0f; + if (value > limit) return limit; + return value; +} + +static void add_rounded(det_point* destination, uint32_t* count, uint32_t capacity, + double x, double y) { + if (*count >= capacity) return; + destination[*count].x = round_half_away(x); + destination[*count].y = round_half_away(y); + ++*count; +} + +static void add_arc(det_point* destination, uint32_t* count, uint32_t capacity, + double center_x, double center_y, double start, double end, double radius) { + const double pi = 3.14159265358979323846; + double radius_abs = fabs(radius); + double chord = radius_abs < 0.25 ? radius_abs : 0.25; + double fraction = fabs(end - start) / (2.0 * pi); + int64_t steps; + int64_t cap; + double x; + double y; + double step; + double cos_step; + double sin_step; + int64_t i; + if (fraction <= 0.0 || radius_abs <= 0.0) return; + steps = (int64_t)(fraction * pi / acos(1.0 - chord / radius_abs)); + if (steps < 2) steps = 2; + cap = (int64_t)(222.0 * fraction); + if (steps > cap) steps = cap > 2 ? cap : 2; + x = cos(start); + y = sin(start); + step = (end - start) / (double)steps; + cos_step = cos(step); + sin_step = sin(step); + for (i = 0; i <= steps; ++i) { + double next_x; + add_rounded(destination, count, capacity, center_x + x * radius, + center_y + y * radius); + next_x = x * cos_step - y * sin_step; + y = x * sin_step + y * cos_step; + x = next_x; + } +} + +/* Convex outward offset with round joins, the same construction as Clipper + * 5.1.5 PolyOffsetBuilder for a single positive-delta polygon (the trailing + * boolean union is unnecessary because the caller takes a min-area rect). */ +static uint32_t offset_round(const det_point* box, uint32_t count, double delta, + lw_db_postprocess_workspace* workspace) { + int64_t* pts = (int64_t*)workspace->unclip_pts; + double* normals = (double*)workspace->unclip_normals; + det_point* result = (det_point*)workspace->unclip_result; + uint32_t result_count = 0u; + uint32_t n = 0u; + double winding = 0.0; + uint32_t prev; + uint32_t i; + for (i = 0u; i < count; ++i) { + int64_t x = round_half_even_i64((double)box[i].x); + int64_t y = round_half_even_i64((double)box[i].y); + if (n > 0u && pts[(size_t)(n - 1u) * 2u] == x && pts[(size_t)(n - 1u) * 2u + 1u] == y) { + continue; + } + pts[(size_t)n * 2u] = x; + pts[(size_t)n * 2u + 1u] = y; + ++n; + } + if (n >= 2u && pts[0] == pts[(size_t)(n - 1u) * 2u] && + pts[1] == pts[(size_t)(n - 1u) * 2u + 1u]) { + --n; + } + if (n < 3u) return 0u; + for (i = 0u; i < n; ++i) { + int64_t ax = pts[(size_t)i * 2u]; + int64_t ay = pts[(size_t)i * 2u + 1u]; + int64_t bx = pts[(size_t)((i + 1u) % n) * 2u]; + int64_t by = pts[(size_t)((i + 1u) % n) * 2u + 1u]; + winding += (double)ax * by - (double)bx * ay; + } + if (winding < 0.0) { + for (i = 0u; i < n / 2u; ++i) { + int64_t tx = pts[(size_t)i * 2u]; + int64_t ty = pts[(size_t)i * 2u + 1u]; + pts[(size_t)i * 2u] = pts[(size_t)(n - 1u - i) * 2u]; + pts[(size_t)i * 2u + 1u] = pts[(size_t)(n - 1u - i) * 2u + 1u]; + pts[(size_t)(n - 1u - i) * 2u] = tx; + pts[(size_t)(n - 1u - i) * 2u + 1u] = ty; + } + } + for (i = 0u; i < n; ++i) { + double dx = (double)(pts[(size_t)((i + 1u) % n) * 2u] - pts[(size_t)i * 2u]); + double dy = (double)(pts[(size_t)((i + 1u) % n) * 2u + 1u] - pts[(size_t)i * 2u + 1u]); + double length = sqrt(dx * dx + dy * dy); + if (length > 0.0) { + dx /= length; + dy /= length; + } + normals[(size_t)i * 2u] = dy; + normals[(size_t)i * 2u + 1u] = -dx; + } + prev = n - 1u; + for (i = 0u; i < n; ++i) { + int64_t x = pts[(size_t)i * 2u]; + int64_t y = pts[(size_t)i * 2u + 1u]; + double incoming_x = normals[(size_t)prev * 2u]; + double incoming_y = normals[(size_t)prev * 2u + 1u]; + double outgoing_x = normals[(size_t)i * 2u]; + double outgoing_y = normals[(size_t)i * 2u + 1u]; + add_rounded(result, &result_count, workspace->unclip_capacity, + (double)x + incoming_x * delta, (double)y + incoming_y * delta); + if ((incoming_x * outgoing_y - outgoing_x * incoming_y) * delta >= 0.0) { + if (outgoing_x * incoming_x + outgoing_y * incoming_y < 0.985) { + double start = atan2(incoming_y, incoming_x); + double end = atan2(outgoing_y, outgoing_x); + if (end < start) end += 2.0 * 3.14159265358979323846; + add_arc(result, &result_count, workspace->unclip_capacity, + (double)x, (double)y, start, end, delta); + } + } else { + add_rounded(result, &result_count, workspace->unclip_capacity, + (double)x, (double)y); + } + add_rounded(result, &result_count, workspace->unclip_capacity, + (double)x + outgoing_x * delta, (double)y + outgoing_y * delta); + prev = i; + } + return result_count; +} + +/* cv2.fillPoly scanline replica: per-row crossing parity against float + * thresholds plus exact integer on-edge hits, over an int32-truncated + * polygon, averaging the prediction map. */ +static float polygon_score(const float* prediction, uint32_t width, uint32_t height, + const det_point* polygon, uint32_t count, + lw_db_postprocess_workspace* workspace) { + float* thresholds = (float*)workspace->score_thresholds; + int32_t* hit_low = (int32_t*)workspace->score_hit_low; + int32_t* hit_high = (int32_t*)workspace->score_hit_high; + int32_t* poly_x = (int32_t*)workspace->score_poly_x; + int32_t* poly_y = (int32_t*)workspace->score_poly_y; + float min_x = INFINITY; + float max_x = -INFINITY; + float min_y = INFINITY; + float max_y = -INFINITY; + int32_t left; + int32_t right; + int32_t top; + int32_t bottom; + double sum = 0.0; + uint64_t pixel_count = 0u; + int32_t y; + uint32_t i; + for (i = 0u; i < count; ++i) { + if (polygon[i].x < min_x) min_x = polygon[i].x; + if (polygon[i].x > max_x) max_x = polygon[i].x; + if (polygon[i].y < min_y) min_y = polygon[i].y; + if (polygon[i].y > max_y) max_y = polygon[i].y; + } + left = (int32_t)floorf(min_x); + right = (int32_t)ceilf(max_x); + top = (int32_t)floorf(min_y); + bottom = (int32_t)ceilf(max_y); + if (left < 0) left = 0; + if (top < 0) top = 0; + if (right >= (int32_t)width) right = (int32_t)width - 1; + if (bottom >= (int32_t)height) bottom = (int32_t)height - 1; + if (left > right || top > bottom) return 0.0f; + for (i = 0u; i < count; ++i) { + poly_x[i] = (int32_t)polygon[i].x - left; + poly_y[i] = (int32_t)polygon[i].y - top; + } + for (y = top; y <= bottom; ++y) { + int32_t py = y - top; + uint32_t threshold_count = 0u; + uint32_t hit_count = 0u; + uint32_t threshold_index = 0u; + int32_t x; + uint32_t j; + for (i = 0u, j = count - 1u; i < count; j = i++) { + int32_t ax = poly_x[i]; + int32_t ay = poly_y[i]; + int32_t bx = poly_x[j]; + int32_t by = poly_y[j]; + int32_t dx = bx - ax; + int32_t dy = by - ay; + if ((ay > py) != (by > py)) { + thresholds[threshold_count++] = (float)dx * (py - ay) / dy + ax; + } + if (py < (ay < by ? ay : by) || py > (ay > by ? ay : by)) continue; + if (dy == 0) { + hit_low[hit_count] = ax < bx ? ax : bx; + hit_high[hit_count] = ax > bx ? ax : bx; + ++hit_count; + } else { + int64_t numerator = (int64_t)(py - ay) * dx; + if (numerator % dy == 0) { + int32_t hx = ax + (int32_t)(numerator / dy); + if (hx >= (ax < bx ? ax : bx) && hx <= (ax > bx ? ax : bx)) { + hit_low[hit_count] = hx; + hit_high[hit_count] = hx; + ++hit_count; + } + } + } + } + for (i = 1u; i < threshold_count; ++i) { + float value = thresholds[i]; + uint32_t insert = i; + while (insert > 0u && thresholds[insert - 1u] > value) { + thresholds[insert] = thresholds[insert - 1u]; + --insert; + } + thresholds[insert] = value; + } + for (x = left; x <= right; ++x) { + int32_t px = x - left; + int inside; + while (threshold_index < threshold_count && thresholds[threshold_index] <= px) { + ++threshold_index; + } + inside = ((threshold_count - threshold_index) & 1u) != 0u; + if (!inside) { + for (i = 0u; i < hit_count; ++i) { + if (px >= hit_low[i] && px <= hit_high[i]) { + inside = 1; + break; + } + } + } + if (inside) { + sum += prediction[(size_t)((uint64_t)(uint32_t)y * width + (uint32_t)x)]; + ++pixel_count; + } + } + } + return pixel_count == 0u ? 0.0f : (float)(sum / (double)pixel_count); +} + +/* Grow the reusable scratch buffers to cover this image. Buffers only grow; + * a detector that processes many same-size images allocates once. */ +static lw_status workspace_grow(lw_db_postprocess_workspace* workspace, uint32_t pixels, + uint32_t max_candidates) { + uint8_t* bitmap = workspace->bitmap; + uint8_t* visited = workspace->visited; + uint8_t* background_visited = workspace->background_visited; + uint8_t* dilated = workspace->dilated; + uint8_t* interior = workspace->interior; + uint32_t* stack = workspace->queue; + det_point* points = (det_point*)workspace->points; + det_point* hull = (det_point*)workspace->hull; + lw_detection_box* results = (lw_detection_box*)workspace->results; + if (workspace->pixel_capacity < pixels) { + bitmap = (uint8_t*)malloc((size_t)pixels); + visited = (uint8_t*)malloc((size_t)pixels); + background_visited = (uint8_t*)malloc((size_t)pixels); + dilated = (uint8_t*)malloc((size_t)pixels); + interior = (uint8_t*)malloc((size_t)pixels); + stack = (uint32_t*)malloc(3u * (size_t)pixels * sizeof(*stack)); + if (bitmap == NULL || visited == NULL || background_visited == NULL || + dilated == NULL || interior == NULL || stack == NULL) { + free(stack); + free(interior); + free(dilated); + free(background_visited); + free(visited); + free(bitmap); + return LW_STATUS_OUT_OF_MEMORY; + } + free(workspace->queue); + free(workspace->interior); + free(workspace->dilated); + free(workspace->background_visited); + free(workspace->visited); + free(workspace->bitmap); + workspace->bitmap = bitmap; + workspace->visited = visited; + workspace->background_visited = background_visited; + workspace->dilated = dilated; + workspace->interior = interior; + workspace->queue = stack; + workspace->pixel_capacity = pixels; + } + if (workspace->point_capacity < pixels) { + points = (det_point*)malloc((size_t)pixels * sizeof(*points)); + hull = (det_point*)malloc((size_t)pixels * 2u * sizeof(*hull)); + if (points == NULL || hull == NULL) { + free(hull); + free(points); + return LW_STATUS_OUT_OF_MEMORY; + } + free(workspace->hull); + free(workspace->points); + workspace->points = points; + workspace->hull = hull; + workspace->point_capacity = pixels; + } + if (workspace->result_capacity < max_candidates) { + results = (lw_detection_box*)calloc(max_candidates, sizeof(*results)); + if (results == NULL) { + return LW_STATUS_OUT_OF_MEMORY; + } + free(workspace->results); + workspace->results = results; + workspace->result_capacity = max_candidates; + } + return LW_STATUS_OK; +} + +/* Grow the polygon scratch (unclip round-join and PolygonScore arrays). */ +static lw_status workspace_grow_polygon(lw_db_postprocess_workspace* workspace, uint32_t count) { + uint32_t capacity; + if (count > (UINT32_MAX - 7u) / 228u) return LW_STATUS_OUT_OF_BOUNDS; + capacity = count * 228u + 8u; + if (workspace->unclip_capacity < capacity) { + int64_t* pts = (int64_t*)malloc((size_t)count * 2u * sizeof(*pts)); + double* normals = (double*)malloc((size_t)count * 2u * sizeof(*normals)); + det_point* result = (det_point*)malloc((size_t)capacity * sizeof(*result)); + if (pts == NULL || normals == NULL || result == NULL) { + free(result); + free(normals); + free(pts); + return LW_STATUS_OUT_OF_MEMORY; + } + free(workspace->unclip_result); + free(workspace->unclip_normals); + free(workspace->unclip_pts); + workspace->unclip_pts = pts; + workspace->unclip_normals = normals; + workspace->unclip_result = result; + workspace->unclip_capacity = capacity; + } + if (workspace->score_capacity < count) { + float* thresholds = (float*)malloc((size_t)count * sizeof(*thresholds)); + int32_t* hit_low = (int32_t*)malloc((size_t)count * sizeof(*hit_low)); + int32_t* hit_high = (int32_t*)malloc((size_t)count * sizeof(*hit_high)); + int32_t* poly_x = (int32_t*)malloc((size_t)count * sizeof(*poly_x)); + int32_t* poly_y = (int32_t*)malloc((size_t)count * sizeof(*poly_y)); + if (thresholds == NULL || hit_low == NULL || hit_high == NULL || + poly_x == NULL || poly_y == NULL) { + free(poly_y); + free(poly_x); + free(hit_high); + free(hit_low); + free(thresholds); + return LW_STATUS_OUT_OF_MEMORY; + } + free(workspace->score_poly_y); + free(workspace->score_poly_x); + free(workspace->score_hit_high); + free(workspace->score_hit_low); + free(workspace->score_thresholds); + workspace->score_thresholds = thresholds; + workspace->score_hit_low = hit_low; + workspace->score_hit_high = hit_high; + workspace->score_poly_x = poly_x; + workspace->score_poly_y = poly_y; + workspace->score_capacity = count; + } + return LW_STATUS_OK; +} + +void lw_db_postprocess_workspace_free(lw_db_postprocess_workspace* workspace) { + if (workspace == NULL) return; + free(workspace->score_poly_y); + free(workspace->score_poly_x); + free(workspace->score_hit_high); + free(workspace->score_hit_low); + free(workspace->score_thresholds); + free(workspace->unclip_result); + free(workspace->unclip_normals); + free(workspace->unclip_pts); + free(workspace->results); + free(workspace->hull); + free(workspace->points); + free(workspace->queue); + free(workspace->interior); + free(workspace->dilated); + free(workspace->background_visited); + free(workspace->visited); + free(workspace->bitmap); + memset(workspace, 0, sizeof(*workspace)); +} + +/* Scanline 8-connected flood fill: the visited set matches the classic + * per-pixel BFS, and the boundary points come from the precomputed + * 4-neighborhood interior bitmap (interior[p]==0 reproduces the per-pixel + * four-neighbor probe exactly). */ +static uint32_t fill_foreground(const uint8_t* bitmap, uint8_t* visited, + uint32_t map_width, uint32_t map_height, + uint32_t start, const uint8_t* interior, + det_point* points, uint32_t* stack) { + uint32_t top = 0u; + uint32_t point_count = 0u; + uint32_t seed_y = start / map_width; + uint32_t seed_x = start % map_width; + uint32_t seed_row = seed_y * map_width; + uint32_t left = seed_x; + uint32_t right = seed_x; + while (left > 0u && bitmap[seed_row + left - 1u] != 0u && + visited[seed_row + left - 1u] == 0u) { + --left; + } + while (right + 1u < map_width && bitmap[seed_row + right + 1u] != 0u && + visited[seed_row + right + 1u] == 0u) { + ++right; + } + memset(visited + seed_row + left, 1u, right - left + 1u); + stack[top++] = seed_y; + stack[top++] = left; + stack[top++] = right; + while (top > 0u) { + uint32_t run_right = stack[--top]; + uint32_t run_left = stack[--top]; + uint32_t y = stack[--top]; + uint32_t row = y * map_width; + uint32_t x; + int32_t direction; + uint32_t scan_from; + uint32_t scan_to; + for (x = run_left; x <= run_right; ++x) { + if (interior[row + x] == 0u) { + points[point_count].x = (float)x; + points[point_count].y = (float)y; + ++point_count; + } + } + scan_from = run_left > 0u ? run_left - 1u : 0u; + scan_to = run_right + 1u < map_width ? run_right + 1u : map_width - 1u; + for (direction = -1; direction <= 1; direction += 2) { + int64_t neighbor_y = (int64_t)y + direction; + uint32_t nrow; + if (neighbor_y < 0 || neighbor_y >= (int64_t)map_height) continue; + nrow = (uint32_t)neighbor_y * map_width; + x = scan_from; + while (x <= scan_to) { + if (bitmap[nrow + x] != 0u && visited[nrow + x] == 0u) { + uint32_t next_left = x; + uint32_t next_right = x; + while (next_left > 0u && bitmap[nrow + next_left - 1u] != 0u && + visited[nrow + next_left - 1u] == 0u) { + --next_left; + } + while (next_right + 1u < map_width && + bitmap[nrow + next_right + 1u] != 0u && + visited[nrow + next_right + 1u] == 0u) { + ++next_right; + } + memset(visited + nrow + next_left, 1u, next_right - next_left + 1u); + stack[top++] = (uint32_t)neighbor_y; + stack[top++] = next_left; + stack[top++] = next_right; + x = next_right + 2u; + } else { + ++x; + } + } + } + } + return point_count; +} + +/* Scanline flood fill over the 4-connected background component containing + * `seed`. With boundary_points != NULL this traces the component's contour + * (OpenCV RETR_LIST background-hole parity); with NULL it only marks the + * visited set. */ +static uint32_t fill_background(const uint8_t* bitmap, uint8_t* visited, + uint32_t map_width, uint32_t map_height, uint32_t seed, + det_point* boundary_points, uint32_t* stack) { + uint32_t top = 0u; + uint32_t boundary_count = 0u; + uint32_t seed_y = seed / map_width; + uint32_t seed_x = seed % map_width; + uint32_t seed_row = seed_y * map_width; + uint32_t left = seed_x; + uint32_t right = seed_x; + if (bitmap[seed] != 0u || visited[seed] != 0u) return 0u; + while (left > 0u && bitmap[seed_row + left - 1u] == 0u && + visited[seed_row + left - 1u] == 0u) { + --left; + } + while (right + 1u < map_width && bitmap[seed_row + right + 1u] == 0u && + visited[seed_row + right + 1u] == 0u) { + ++right; + } + memset(visited + seed_row + left, 1u, right - left + 1u); + stack[top++] = seed_y; + stack[top++] = left; + stack[top++] = right; + while (top > 0u) { + uint32_t run_right = stack[--top]; + uint32_t run_left = stack[--top]; + uint32_t y = stack[--top]; + uint32_t row = y * map_width; + int32_t direction; + if (boundary_points != NULL) { + if (run_left > 0u && bitmap[row + run_left - 1u] != 0u) { + boundary_points[boundary_count].x = (float)run_left; + boundary_points[boundary_count].y = (float)y; + ++boundary_count; + } + if (run_right + 1u < map_width && bitmap[row + run_right + 1u] != 0u) { + boundary_points[boundary_count].x = (float)run_right; + boundary_points[boundary_count].y = (float)y; + ++boundary_count; + } + } + for (direction = -1; direction <= 1; direction += 2) { + int64_t neighbor_y = (int64_t)y + direction; + uint32_t nrow; + uint32_t x; + if (neighbor_y < 0 || neighbor_y >= (int64_t)map_height) continue; + nrow = (uint32_t)neighbor_y * map_width; + x = run_left; + while (x <= run_right) { + if (bitmap[nrow + x] != 0u) { + if (boundary_points != NULL) { + boundary_points[boundary_count].x = (float)x; + boundary_points[boundary_count].y = (float)y; + ++boundary_count; + } + ++x; + } else if (visited[nrow + x] == 0u) { + uint32_t next_left = x; + uint32_t next_right = x; + while (next_left > 0u && bitmap[nrow + next_left - 1u] == 0u && + visited[nrow + next_left - 1u] == 0u) { + --next_left; + } + while (next_right + 1u < map_width && + bitmap[nrow + next_right + 1u] == 0u && + visited[nrow + next_right + 1u] == 0u) { + ++next_right; + } + memset(visited + nrow + next_left, 1u, next_right - next_left + 1u); + stack[top++] = (uint32_t)neighbor_y; + stack[top++] = next_left; + stack[top++] = next_right; + x = next_right + 1u; + } else { + ++x; + } + } + } + } + return boundary_count; +} + +/* Reference TryBuildDetection: min-area rect of the contour -> PolygonScore + * on the mini-box -> Clipper-style round-join unclip -> min-area rect of the + * expanded polygon -> round-half-even remap to source coordinates -> min-area + * rect of the mapped corners (PaddleX CropByPolys parity). */ +static int try_build_detection(const float* prediction, uint32_t map_width, uint32_t map_height, + float box_threshold, float unclip_ratio, + uint32_t source_width, uint32_t source_height, + det_point* boundary, uint32_t boundary_count, + det_point* hull, lw_db_postprocess_workspace* workspace, + lw_pipeline_component_profile* profile, + lw_detection_box* out_box) { + det_rectangle rectangle; + det_point mini_box[4]; + det_point corners[4]; + det_point mapped[4]; + float shortest_side; + float score; + double area_value; + double perimeter; + double distance_value; + uint32_t unclipped_count; + uint32_t i; + if (!minimum_rectangle(hull, convex_hull(boundary, boundary_count, hull), &rectangle)) { + return 0; + } + shortest_side = rectangle.max_u - rectangle.min_u < rectangle.max_v - rectangle.min_v + ? rectangle.max_u - rectangle.min_u + : rectangle.max_v - rectangle.min_v; + if (shortest_side < 3.0f) return 0; + rectangle_points(&rectangle, 0.0f, mini_box); + order_clockwise(mini_box); + if (workspace_grow_polygon(workspace, 4u) != LW_STATUS_OK) return 0; + score = polygon_score(prediction, map_width, map_height, mini_box, 4u, workspace); + if (!isfinite(score) || score < box_threshold) return 0; + area_value = fabs(signed_area(mini_box, 4u)); + perimeter = 0.0; + for (i = 0u; i < 4u; ++i) { + float dx = mini_box[(i + 1u) % 4u].x - mini_box[i].x; + float dy = mini_box[(i + 1u) % 4u].y - mini_box[i].y; + perimeter += sqrtf(dx * dx + dy * dy); + } + distance_value = area_value * (double)unclip_ratio / perimeter; + if (!isfinite(distance_value) || distance_value <= 0.0) return 0; + { + uint64_t unclip_started = lw_pipeline_profile_now(profile); + unclipped_count = offset_round(mini_box, 4u, distance_value, workspace); + lw_pipeline_profile_add_elapsed(profile == NULL ? NULL : &profile->unclip_nanoseconds, + unclip_started, profile); + } + /* The grow(4) capacity already covers the round-join output bound + * (4 corners x 228 points); do not grow again, the buffer must stay in + * place because convex_hull sorts its input in place. */ + { + det_point* unclipped = (det_point*)workspace->unclip_result; + if (!minimum_rectangle(hull, convex_hull(unclipped, unclipped_count, hull), + &rectangle)) { + return 0; + } + } + shortest_side = rectangle.max_u - rectangle.min_u < rectangle.max_v - rectangle.min_v + ? rectangle.max_u - rectangle.min_u + : rectangle.max_v - rectangle.min_v; + rectangle_points(&rectangle, 0.0f, corners); + order_clockwise(corners); + /* PaddleOCR's quad post-process maps coordinates back to the source with + * np.round (half-even) and clips to [0, size]. */ + for (i = 0u; i < 4u; ++i) { + corners[i].x = clamp_coord( + round_half_even_f(corners[i].x / (float)map_width * (float)source_width), + (float)source_width); + corners[i].y = clamp_coord( + round_half_even_f(corners[i].y / (float)map_height * (float)source_height), + (float)source_height); + } + /* PaddleX CropByPolys converts the mapped polygon to int32 and calls + * minAreaRect before perspective cropping; reproduce that geometry. */ + memcpy(mapped, corners, sizeof(mapped)); + if (!minimum_rectangle(hull, convex_hull(mapped, 4u, hull), &rectangle)) return 0; + rectangle_points(&rectangle, 0.0f, corners); + order_clockwise(corners); + if (hypotf(corners[0].x - corners[1].x, corners[0].y - corners[1].y) <= 4.0f || + hypotf(corners[0].x - corners[3].x, corners[0].y - corners[3].y) <= 4.0f) { + return 0; + } + out_box->x1 = corners[0].x; + out_box->y1 = corners[0].y; + out_box->x2 = corners[1].x; + out_box->y2 = corners[1].y; + out_box->x3 = corners[2].x; + out_box->y3 = corners[2].y; + out_box->x4 = corners[3].x; + out_box->y4 = corners[3].y; + out_box->score = score; + return 1; +} + +lw_status lw_db_postprocess_f32_ws( + const float* prediction, uint32_t map_width, uint32_t map_height, + float bitmap_threshold, float box_threshold, float unclip_ratio, + uint32_t use_dilation, uint32_t max_candidates, uint32_t source_width, + uint32_t source_height, float width_ratio, float height_ratio, + lw_detection_box* boxes, uint32_t box_capacity, uint32_t* box_count, + lw_db_postprocess_workspace* workspace, lw_pipeline_component_profile* profile) { + uint64_t pixel_count; + uint8_t* bitmap; + uint8_t* visited; + uint8_t* interior; + uint32_t* stack; + det_point* points; + det_point* hull; + lw_detection_box* results; + uint32_t result_count = 0u; + uint32_t candidate_count = 0u; + uint32_t start; + lw_status status; + if (box_count != NULL) + *box_count = 0u; + if (prediction == NULL || box_count == NULL || map_width == 0u || map_height == 0u || + source_width == 0u || source_height == 0u || max_candidates == 0u || use_dilation > 1u || + !isfinite(bitmap_threshold) || bitmap_threshold < 0.0f || bitmap_threshold > 1.0f || + !isfinite(box_threshold) || box_threshold < 0.0f || box_threshold > 1.0f || + !isfinite(unclip_ratio) || unclip_ratio <= 0.0f || !isfinite(width_ratio) || + width_ratio <= 0.0f || !isfinite(height_ratio) || height_ratio <= 0.0f || + (boxes == NULL && box_capacity != 0u) || workspace == NULL) { + return LW_STATUS_INVALID_ARGUMENT; + } + if (map_width > INT32_MAX || map_height > INT32_MAX) { + return LW_STATUS_OUT_OF_BOUNDS; + } + pixel_count = (uint64_t)map_width * map_height; + if (pixel_count > UINT32_MAX || + pixel_count > SIZE_MAX / (3u * sizeof(*stack)) || + pixel_count > SIZE_MAX / sizeof(det_point) || + !allocation_fits(max_candidates, sizeof(*results))) { + return LW_STATUS_OUT_OF_BOUNDS; + } + status = workspace_grow(workspace, (uint32_t)pixel_count, max_candidates); + if (status != LW_STATUS_OK) + return status; + bitmap = workspace->bitmap; + visited = workspace->visited; + interior = workspace->interior; + stack = workspace->queue; + points = (det_point*)workspace->points; + hull = (det_point*)workspace->hull; + results = (lw_detection_box*)workspace->results; + memset(visited, 0, (size_t)pixel_count); + /* Convert model probabilities to a compact bitmap after rejecting NaN/Inf. */ +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_DB_BITMAP_SIMD) + if (lw_wasm128_threshold_bitmap_f32(prediction, bitmap, pixel_count, + bitmap_threshold) != 0) { +#else + if (lw_avx2_threshold_bitmap_f32(prediction, bitmap, pixel_count, + bitmap_threshold) != 0) { +#endif + return LW_STATUS_INVALID_ARGUMENT; + } + if (use_dilation != 0u) { + /* OpenCV's default anchor for a 2x2 kernel is (0, 0): the source + * footprint for dst(x,y) is (x,y), (x+1,y), (x,y+1), (x+1,y+1). */ + uint8_t* dilated = workspace->dilated; + memset(dilated, 0, (size_t)pixel_count); + for (start = 0u; start < pixel_count; ++start) { + if (bitmap[(size_t)start] != 0u) { + uint32_t x = start % map_width; + uint32_t y = start / map_width; + dilated[(size_t)start] = 1u; + if (x + 1u < map_width) + dilated[(size_t)(start + 1u)] = 1u; + if (y + 1u < map_height) + dilated[(size_t)(start + map_width)] = 1u; + if (x + 1u < map_width && y + 1u < map_height) + dilated[(size_t)(start + map_width + 1u)] = 1u; + } + } + bitmap = dilated; + } + lw_avx2_interior_bitmap_u8(bitmap, interior, map_width, map_height); + /* Flood-fill each 8-connected foreground component; only boundary pixels + * are retained because the convex hull does not need the interior. */ + for (start = 0u; start < pixel_count && candidate_count < max_candidates; ++start) { + uint32_t point_count; + if (bitmap[(size_t)start] == 0u || visited[(size_t)start] != 0u) + continue; + ++candidate_count; + point_count = fill_foreground(bitmap, visited, map_width, map_height, start, interior, + points, stack); + if (try_build_detection(prediction, map_width, map_height, box_threshold, unclip_ratio, + source_width, source_height, points, point_count, hull, workspace, + profile, &results[result_count])) { + ++result_count; + } + } + /* OpenCV RETR_LIST also returns contours for enclosed background regions: + * flood the exterior background from the image border, then trace the + * remaining zero components as holes with 4-connectivity (the complement + * of the 8-connected foreground contours). */ + memset(workspace->background_visited, 0, (size_t)pixel_count); + for (start = 0u; start < map_width; ++start) { + (void)fill_background(bitmap, workspace->background_visited, map_width, map_height, + start, NULL, stack); + if (map_height > 1u) { + (void)fill_background(bitmap, workspace->background_visited, map_width, map_height, + (map_height - 1u) * map_width + start, NULL, stack); + } + } + for (start = 1u; start + 1u < map_height; ++start) { + (void)fill_background(bitmap, workspace->background_visited, map_width, map_height, + start * map_width, NULL, stack); + if (map_width > 1u) { + (void)fill_background(bitmap, workspace->background_visited, map_width, map_height, + start * map_width + map_width - 1u, NULL, stack); + } + } + for (start = 0u; start < pixel_count && candidate_count < max_candidates; ++start) { + uint32_t point_count; + if (bitmap[(size_t)start] != 0u || workspace->background_visited[(size_t)start] != 0u) { + continue; + } + ++candidate_count; + point_count = fill_background(bitmap, workspace->background_visited, map_width, + map_height, start, points, stack); + if (try_build_detection(prediction, map_width, map_height, box_threshold, unclip_ratio, + source_width, source_height, points, point_count, hull, workspace, + profile, &results[result_count])) { + ++result_count; + } + } + *box_count = result_count; + if (boxes != NULL) { + if (box_capacity < result_count) { + return LW_STATUS_OUT_OF_BOUNDS; + } + memcpy(boxes, results, (size_t)result_count * sizeof(*boxes)); + } + return LW_STATUS_OK; +} + +lw_status lw_db_postprocess_f32(const float* prediction, uint32_t map_width, uint32_t map_height, + float bitmap_threshold, float box_threshold, float unclip_ratio, + uint32_t use_dilation, uint32_t max_candidates, + uint32_t source_width, uint32_t source_height, float width_ratio, + float height_ratio, lw_detection_box* boxes, uint32_t box_capacity, + uint32_t* box_count) { + lw_db_postprocess_workspace workspace; + lw_status status; + memset(&workspace, 0, sizeof(workspace)); + status = lw_db_postprocess_f32_ws(prediction, map_width, map_height, bitmap_threshold, + box_threshold, unclip_ratio, use_dilation, max_candidates, + source_width, source_height, width_ratio, height_ratio, boxes, + box_capacity, box_count, &workspace, NULL); + lw_db_postprocess_workspace_free(&workspace); + return status; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/det_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/det_internal.h new file mode 100644 index 00000000..96a6cc01 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/det_internal.h @@ -0,0 +1,107 @@ +#ifndef LW_DET_INTERNAL_H +#define LW_DET_INTERNAL_H + +/* Private boundary between DET preprocessing, inference and postprocessing. */ + +#include "lw_infer.h" +#include "reading_order.h" + +#include + +typedef struct lw_pipeline_component_profile lw_pipeline_component_profile; + +lw_status lw_det_compute_size(uint32_t source_width, uint32_t source_height, + uint32_t limit_side_length, uint32_t* resized_width, + uint32_t* resized_height, float* width_ratio, float* height_ratio); + +lw_status lw_det_preprocess_bgr_u8(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, uint32_t resized_width, + uint32_t resized_height, float* output, + uint64_t output_element_count); + +/* OpenCV-parity fixed-point resize + LUT normalization, with optional row + * sharding over the detector's intra-op pool. */ +typedef struct lw_det_preprocess_workspace { + int32_t* x_offsets; + int16_t* x_coefficients; + int32_t* rows; /* (workers + 1) x 2 x width x 3 row scratch */ + uint32_t width; + uint32_t row_slot_count; +} lw_det_preprocess_workspace; +typedef struct lw_thread_pool lw_thread_pool; + +lw_status lw_det_preprocess_bgr_u8_fixed( + const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, uint32_t resized_width, + uint32_t resized_height, float* output, uint64_t output_element_count, + lw_det_preprocess_workspace* workspace, lw_thread_pool* pool, + uint32_t intra_op_thread_count); +lw_status lw_det_preprocess_bgr_u8_fixed_nhwc( + const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, uint32_t resized_width, + uint32_t resized_height, float* output, uint64_t output_element_count, + lw_det_preprocess_workspace* workspace, lw_thread_pool* pool, + uint32_t intra_op_thread_count); + +void lw_det_preprocess_workspace_init(lw_det_preprocess_workspace* workspace); +void lw_det_preprocess_workspace_free(lw_det_preprocess_workspace* workspace); + +lw_status lw_db_postprocess_f32(const float* prediction, uint32_t map_width, uint32_t map_height, + float bitmap_threshold, float box_threshold, float unclip_ratio, + uint32_t use_dilation, uint32_t max_candidates, + uint32_t source_width, uint32_t source_height, float width_ratio, + float height_ratio, lw_detection_box* boxes, uint32_t box_capacity, + uint32_t* box_count); + +/* Reusable grow-only postprocess scratch owned by the detector handle; kills + * the per-image malloc/free churn of the standalone entry. */ +typedef struct lw_db_postprocess_workspace { + uint8_t* bitmap; + uint8_t* visited; + uint8_t* background_visited; + uint8_t* dilated; + uint8_t* interior; + uint32_t* queue; + void* points; + void* hull; + void* results; + void* unclip_pts; + void* unclip_normals; + void* unclip_result; + void* score_thresholds; + void* score_hit_low; + void* score_hit_high; + void* score_poly_x; + void* score_poly_y; + uint32_t point_capacity; + uint32_t hull_capacity; + uint32_t pixel_capacity; + uint32_t result_capacity; + uint32_t unclip_capacity; + uint32_t score_capacity; +} lw_db_postprocess_workspace; + +lw_status lw_db_postprocess_f32_ws( + const float* prediction, uint32_t map_width, uint32_t map_height, + float bitmap_threshold, float box_threshold, float unclip_ratio, + uint32_t use_dilation, uint32_t max_candidates, uint32_t source_width, + uint32_t source_height, float width_ratio, float height_ratio, + lw_detection_box* boxes, uint32_t box_capacity, uint32_t* box_count, + lw_db_postprocess_workspace* workspace, lw_pipeline_component_profile* profile); + +void lw_db_postprocess_workspace_free(lw_db_postprocess_workspace* workspace); + +/* Full OCR gives DET an independent phase-local intra-op budget. */ +void lw_detector_set_intra_op_thread_count(lw_detector* detector, uint32_t thread_count); +uint32_t lw_detector_get_intra_op_thread_count(const lw_detector* detector); + +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +/* Test-only internal hooks: opt the detector into the compiled x64 DET + * backend (the production default stays canonical until the promotion gate + * flips it). */ +void lw_detector_test_enable_x64_backend(lw_detector* detector); +void lw_detector_test_disable_x64_backend(lw_detector* detector); +#endif + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/det_preprocess.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/det_preprocess.c new file mode 100644 index 00000000..970e6aff --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/det_preprocess.c @@ -0,0 +1,128 @@ +#include "det_internal.h" + +/* Resize, pad and normalize BGR8 input into the DET model's NCHW tensor. */ + +#include +#include +#include +#include + +static uint32_t clamp_coordinate(int64_t coordinate, uint32_t limit) { + if (coordinate < 0) { + return 0u; + } + if ((uint64_t)coordinate >= limit) { + return limit - 1u; + } + return (uint32_t)coordinate; +} + +lw_status lw_det_compute_size(uint32_t source_width, uint32_t source_height, + uint32_t limit_side_length, uint32_t* resized_width, + uint32_t* resized_height, float* width_ratio, float* height_ratio) { + uint32_t maximum_side; + double ratio; + double scaled_width; + double scaled_height; + uint64_t rounded_width; + uint64_t rounded_height; + if (resized_width == NULL || resized_height == NULL || width_ratio == NULL || + height_ratio == NULL || source_width == 0u || source_height == 0u || + limit_side_length < 32u || limit_side_length > INT32_MAX) { + return LW_STATUS_INVALID_ARGUMENT; + } + maximum_side = source_width > source_height ? source_width : source_height; + ratio = maximum_side > limit_side_length ? (double)limit_side_length / maximum_side : 1.0; + + /* DET down-sampling stages require dimensions divisible by 32. Rounding + * rather than padding keeps the probability map aligned with the image. */ + scaled_width = (double)source_width * ratio / 32.0; + scaled_height = (double)source_height * ratio / 32.0; + rounded_width = (uint64_t)floor(scaled_width + 0.5) * 32u; + rounded_height = (uint64_t)floor(scaled_height + 0.5) * 32u; + if (rounded_width < 32u) { + rounded_width = 32u; + } + if (rounded_height < 32u) { + rounded_height = 32u; + } + if (rounded_width > INT32_MAX || rounded_height > INT32_MAX) { + return LW_STATUS_OUT_OF_BOUNDS; + } + *resized_width = (uint32_t)rounded_width; + *resized_height = (uint32_t)rounded_height; + *width_ratio = (float)((double)rounded_width / source_width); + *height_ratio = (float)((double)rounded_height / source_height); + return LW_STATUS_OK; +} + +lw_status lw_det_preprocess_bgr_u8(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, uint32_t resized_width, + uint32_t resized_height, float* output, + uint64_t output_element_count) { + static const double mean[3] = {0.485, 0.456, 0.406}; + static const double inverse_std[3] = {1.0 / 0.229, 1.0 / 0.224, 1.0 / 0.225}; + uint64_t row_bytes; + uint64_t required_source_bytes; + uint64_t plane; + uint64_t required_output_elements; + uint32_t output_y; + if (source == NULL || output == NULL || source_width == 0u || source_height == 0u || + resized_width == 0u || resized_height == 0u) { + return LW_STATUS_INVALID_ARGUMENT; + } + row_bytes = (uint64_t)source_width * 3u; + if (row_bytes > UINT32_MAX || source_stride < row_bytes || + (uint64_t)(source_height - 1u) * source_stride > UINT64_MAX - row_bytes) { + return LW_STATUS_INVALID_SHAPE; + } + required_source_bytes = (uint64_t)(source_height - 1u) * source_stride + row_bytes; + plane = (uint64_t)resized_width * resized_height; + if (plane > UINT64_MAX / 3u) { + return LW_STATUS_OUT_OF_BOUNDS; + } + required_output_elements = plane * 3u; + if (required_source_bytes > SIZE_MAX || source_byte_count < required_source_bytes || + output_element_count != required_output_elements || + required_output_elements > SIZE_MAX / sizeof(float)) { + return LW_STATUS_INVALID_SHAPE; + } + /* Half-pixel coordinates match common image-resize implementations. The + * source remains BGR, while the channel planes are written as NCHW. */ + for (output_y = 0u; output_y < resized_height; ++output_y) { + double source_y = ((double)output_y + 0.5) * source_height / resized_height - 0.5; + int64_t source_y0_raw = (int64_t)floor(source_y); + int64_t source_y1_raw = source_y0_raw + 1; + uint32_t source_y0 = clamp_coordinate(source_y0_raw, source_height); + uint32_t source_y1 = clamp_coordinate(source_y1_raw, source_height); + double weight_y = source_y - source_y0_raw; + uint32_t output_x; + for (output_x = 0u; output_x < resized_width; ++output_x) { + double source_x = ((double)output_x + 0.5) * source_width / resized_width - 0.5; + int64_t source_x0_raw = (int64_t)floor(source_x); + int64_t source_x1_raw = source_x0_raw + 1; + uint32_t source_x0 = clamp_coordinate(source_x0_raw, source_width); + uint32_t source_x1 = clamp_coordinate(source_x1_raw, source_width); + double weight_x = source_x - source_x0_raw; + uint32_t channel; + for (channel = 0u; channel < 3u; ++channel) { + double top_left = source[(size_t)((uint64_t)source_y0 * source_stride + + (uint64_t)source_x0 * 3u + channel)]; + double top_right = source[(size_t)((uint64_t)source_y0 * source_stride + + (uint64_t)source_x1 * 3u + channel)]; + double bottom_left = source[(size_t)((uint64_t)source_y1 * source_stride + + (uint64_t)source_x0 * 3u + channel)]; + double bottom_right = source[(size_t)((uint64_t)source_y1 * source_stride + + (uint64_t)source_x1 * 3u + channel)]; + double top = top_left + (top_right - top_left) * weight_x; + double bottom = bottom_left + (bottom_right - bottom_left) * weight_x; + double value = (top + (bottom - top) * weight_y) / 255.0; + output[(size_t)((uint64_t)channel * plane + (uint64_t)output_y * resized_width + + output_x)] = + (float)((value - mean[channel]) * inverse_std[channel]); + } + } + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/det_preprocess_fixed.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/det_preprocess_fixed.c new file mode 100644 index 00000000..63891c15 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/det_preprocess_fixed.c @@ -0,0 +1,243 @@ +#include "det_internal.h" +#include "resize_fixed_internal.h" + +/* OpenCV-parity fixed-point DET resize + ImageNet LUT normalization. The + * 11-bit coefficient path reproduces OpenCV's INTER_LINEAR 8-bit result + * before normalization, so threshold-boundary detector pixels agree with the + * reference implementation. Destination rows are independent; with enough + * rows per worker they are sharded over the detector's intra-op pool. */ + +#include "parallel_internal.h" + +#include +#include +#include +#include +#include + +#define LW_FIXED_MIN_ROWS_PER_WORKER 24u + +typedef struct fixed_row_context { + const uint8_t* source; + uint32_t source_stride; + uint32_t source_width; + uint32_t source_height; + uint32_t resized_width; + uint32_t resized_height; + const int32_t* x_offsets; + const int16_t* x_coefficients; + int32_t* row_scratch; /* per-worker pairs, strided */ + const float* lut; + float* output; + uint32_t plane; + uint8_t nhwc; +} fixed_row_context; + +static const double det_mean[3] = {0.485, 0.456, 0.406}; +static const double det_inverse_std[3] = {1.0 / 0.229, 1.0 / 0.224, 1.0 / 0.225}; + +/* One destination row: two horizontal passes into the worker row scratch, + * then the vertical fixed-point blend and the normalization LUT. */ +static void fixed_row(const fixed_row_context* context, uint32_t oy, int32_t* row0, + int32_t* row1) { + int32_t source_y; + int16_t beta0; + int16_t beta1; + uint32_t source_y0; + uint32_t source_y1; + uint32_t ox; + uint32_t destination = oy * context->resized_width; + lw_fixed_get_linear_coordinate(oy, context->source_height, context->resized_height, + &source_y, &beta0, &beta1); + source_y0 = source_y < 0 ? 0u + : ((uint32_t)source_y >= context->source_height + ? context->source_height - 1u + : (uint32_t)source_y); + { + int32_t clamped = source_y + 1; + source_y1 = clamped < 0 ? 0u + : ((uint32_t)clamped >= context->source_height + ? context->source_height - 1u + : (uint32_t)clamped); + } + lw_fixed_build_horizontal_row(context->source, context->source_stride, context->source_width, + source_y0, context->resized_width, context->x_offsets, + context->x_coefficients, row0); + lw_fixed_build_horizontal_row(context->source, context->source_stride, context->source_width, + source_y1, context->resized_width, context->x_offsets, + context->x_coefficients, row1); + for (ox = 0u; ox < context->resized_width; ++ox) { + uint32_t channel; + uint32_t row_offset = ox * 3u; + for (channel = 0u; channel < 3u; ++channel) { + int32_t value = lw_fixed_blend(row0[row_offset + channel], row1[row_offset + channel], + beta0, beta1); + size_t output_index = context->nhwc != 0u + ? ((size_t)destination + ox) * 3u + channel + : (size_t)channel * context->plane + destination + ox; + context->output[output_index] = + context->lut[(size_t)channel * 256u + (uint32_t)value]; + } + } +} + +static void fixed_row_worker(void* context_void, uint32_t worker_index, uint32_t worker_count) { + fixed_row_context* context = (fixed_row_context*)context_void; + int32_t* row0 = context->row_scratch + + (size_t)worker_index * 2u * context->resized_width * 3u; + int32_t* row1 = row0 + (size_t)context->resized_width * 3u; + uint32_t oy; + (void)worker_count; + for (oy = worker_index; oy < context->resized_height; oy += worker_count) { + fixed_row(context, oy, row0, row1); + } +} + +static void workspace_ensure(lw_det_preprocess_workspace* workspace, uint32_t width, + uint32_t workers) { + uint32_t slot_count = workers + 1u; + if (workspace->width != width || workspace->row_slot_count < slot_count) { + int32_t* x_offsets = (int32_t*)malloc((size_t)width * sizeof(*x_offsets)); + int16_t* x_coefficients = (int16_t*)malloc((size_t)width * 2u * sizeof(*x_coefficients)); + int32_t* rows = (int32_t*)malloc((size_t)slot_count * 2u * width * 3u * sizeof(*rows)); + if (x_offsets == NULL || x_coefficients == NULL || rows == NULL) { + free(rows); + free(x_coefficients); + free(x_offsets); + return; + } + free(workspace->rows); + free(workspace->x_coefficients); + free(workspace->x_offsets); + workspace->x_offsets = x_offsets; + workspace->x_coefficients = x_coefficients; + workspace->rows = rows; + workspace->width = width; + workspace->row_slot_count = slot_count; + } +} + +static void workspace_free(lw_det_preprocess_workspace* workspace) { + if (workspace == NULL) return; + free(workspace->rows); + free(workspace->x_coefficients); + free(workspace->x_offsets); + memset(workspace, 0, sizeof(*workspace)); +} + +/* The 768-entry ImageNet LUT is built with the same double expressions as + * the legacy path, so normalization stays bit-identical for any byte; only + * the resize rounding differs (OpenCV parity). */ +static void build_normalized_lut(float* lut) { + uint32_t channel; + for (channel = 0u; channel < 3u; ++channel) { + uint32_t value; + for (value = 0u; value < 256u; ++value) { + lut[(size_t)channel * 256u + value] = + (float)(((double)value / 255.0 - det_mean[channel]) * + det_inverse_std[channel]); + } + } +} + +static lw_status preprocess_fixed_impl( + const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, uint32_t resized_width, + uint32_t resized_height, float* output, uint64_t output_element_count, + lw_det_preprocess_workspace* workspace, lw_thread_pool* pool, + uint32_t intra_op_thread_count, uint8_t nhwc) { + uint64_t row_bytes; + uint64_t required_source_bytes; + uint64_t plane; + uint64_t required_output_elements; + uint32_t workers = 1u; + fixed_row_context context; + float lut[3u * 256u]; + if (source == NULL || output == NULL || workspace == NULL || source_width == 0u || + source_height == 0u || resized_width == 0u || resized_height == 0u) { + return LW_STATUS_INVALID_ARGUMENT; + } + row_bytes = (uint64_t)source_width * 3u; + if (row_bytes > UINT32_MAX || source_stride < row_bytes || + (uint64_t)(source_height - 1u) * source_stride > UINT64_MAX - row_bytes) { + return LW_STATUS_INVALID_SHAPE; + } + required_source_bytes = (uint64_t)(source_height - 1u) * source_stride + row_bytes; + plane = (uint64_t)resized_width * resized_height; + if (plane > UINT64_MAX / 3u) { + return LW_STATUS_OUT_OF_BOUNDS; + } + required_output_elements = plane * 3u; + if (required_source_bytes > SIZE_MAX || source_byte_count < required_source_bytes || + output_element_count != required_output_elements || + required_output_elements > SIZE_MAX / sizeof(float)) { + return LW_STATUS_INVALID_SHAPE; + } + if (pool != NULL && intra_op_thread_count > 1u && + resized_height / LW_FIXED_MIN_ROWS_PER_WORKER >= 2u) { + workers = intra_op_thread_count; + if (workers > resized_height / LW_FIXED_MIN_ROWS_PER_WORKER) { + workers = resized_height / LW_FIXED_MIN_ROWS_PER_WORKER; + } + if (workers < 2u) workers = 1u; + } + /* Serial pre-allocation: the parallel body must not grow shared buffers. */ + workspace_ensure(workspace, resized_width, workers); + if (workspace->x_offsets == NULL || workspace->rows == NULL) { + return LW_STATUS_OUT_OF_MEMORY; + } + lw_fixed_build_linear_coefficients(source_width, resized_width, workspace->x_offsets, + workspace->x_coefficients); + build_normalized_lut(lut); + context.source = source; + context.source_stride = source_stride; + context.source_width = source_width; + context.source_height = source_height; + context.resized_width = resized_width; + context.resized_height = resized_height; + context.x_offsets = workspace->x_offsets; + context.x_coefficients = workspace->x_coefficients; + context.row_scratch = workspace->rows; + context.lut = lut; + context.output = output; + context.plane = (uint32_t)plane; + context.nhwc = nhwc; + if (workers > 1u) { + uint32_t worker; + lw_thread_pool_run(pool, workers, fixed_row_worker, &context); + (void)worker; + } else { + fixed_row_worker(&context, 0u, 1u); + } + return LW_STATUS_OK; +} + +lw_status lw_det_preprocess_bgr_u8_fixed( + const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, uint32_t resized_width, + uint32_t resized_height, float* output, uint64_t output_element_count, + lw_det_preprocess_workspace* workspace, lw_thread_pool* pool, + uint32_t intra_op_thread_count) { + return preprocess_fixed_impl(source, source_byte_count, source_width, + source_height, source_stride, resized_width, resized_height, output, + output_element_count, workspace, pool, intra_op_thread_count, 0u); +} + +lw_status lw_det_preprocess_bgr_u8_fixed_nhwc( + const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, uint32_t resized_width, + uint32_t resized_height, float* output, uint64_t output_element_count, + lw_det_preprocess_workspace* workspace, lw_thread_pool* pool, + uint32_t intra_op_thread_count) { + return preprocess_fixed_impl(source, source_byte_count, source_width, + source_height, source_stride, resized_width, resized_height, output, + output_element_count, workspace, pool, intra_op_thread_count, 1u); +} + +void lw_det_preprocess_workspace_free(lw_det_preprocess_workspace* workspace) { + workspace_free(workspace); +} + +void lw_det_preprocess_workspace_init(lw_det_preprocess_workspace* workspace) { + if (workspace != NULL) memset(workspace, 0, sizeof(*workspace)); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/detector.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/detector.c new file mode 100644 index 00000000..cab41250 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/detector.c @@ -0,0 +1,748 @@ +#include "lw_infer.h" +#include "abi_compat_internal.h" + +/* Public DET handle: preprocessing -> graph execution -> DB postprocessing. */ + +#include "det_internal.h" +#include "error_internal.h" +#include "executor_internal.h" +#include "lwm_read.h" +#include "model_internal.h" +#include "operator_internal.h" +#include "profile_internal.h" +#include "session_internal.h" + +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +#include "x64_det_backend_internal.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#define LW_DET_DEFAULT_LIMIT_SIDE_LENGTH 960u +#define LW_DET_MAX_LIMIT_SIDE_LENGTH 4096u +#define LW_DET_DEFAULT_MAX_CANDIDATES 1000u +#define LW_DET_MAX_CANDIDATES 10000u +#define LW_DET_DEFAULT_MAX_IMAGE_PIXELS UINT64_C(40000000) + +struct lw_detector { + lw_model* model; + lw_session* session; + float* input; + float* probabilities; + uint64_t input_element_count; + uint64_t probability_element_count; + uint32_t resized_width; + uint32_t resized_height; + uint32_t intra_op_thread_count; + uint32_t reading_order; + lw_session_options session_options; + lw_detector_info info; + lw_db_postprocess_workspace postprocess_workspace; + lw_det_preprocess_workspace preprocess_workspace; + uint8_t* padded_source; + uint32_t padded_width; + uint32_t padded_height; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_det_program* x64_program; + lw_x64_det_instance* x64_instance; + uint8_t x64_backend_enabled; +#endif +}; + +static void clear_result(lw_detection_result* result) { + memset(result, 0, sizeof(*result)); + result->struct_size = (uint32_t)sizeof(*result); +} + +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) +/* CI-only diagnostics for the compiled DET path. Never scan tensors during + * ordinary benchmark measurements or in the browser. */ +static void detector_debug_tensor(const char* stage, const float* values, uint64_t count) { + float minimum = INFINITY; + float maximum = -INFINITY; + uint64_t finite_count = 0u; + uint64_t above_threshold = 0u; + uint64_t index; + for (index = 0u; index < count; ++index) { + float value = values[index]; + if (!isfinite(value)) continue; + ++finite_count; + if (value < minimum) minimum = value; + if (value > maximum) maximum = value; + if (value > 0.3f) ++above_threshold; + } + if (finite_count == 0u) { + minimum = 0.0f; + maximum = 0.0f; + } + (void)fprintf(stderr, + "LW_WASM_DET_DEBUG stage=%s count=%llu finite=%llu above_0.3=%llu " + "min=%.6g max=%.6g first=%.6g\n", + stage, (unsigned long long)count, (unsigned long long)finite_count, + (unsigned long long)above_threshold, (double)minimum, (double)maximum, + count == 0u ? 0.0 : (double)values[0]); +} +#endif + +void lw_detector_options_init(lw_detector_options* options) { + lw_model_options model_options; + lw_session_options session_options; + if (options == NULL) + return; + lw_model_options_init(&model_options); + lw_session_options_init(&session_options); + memset(options, 0, sizeof(*options)); + options->struct_size = (uint32_t)sizeof(*options); + options->limit_side_length = LW_DET_DEFAULT_LIMIT_SIDE_LENGTH; + options->max_candidates = LW_DET_DEFAULT_MAX_CANDIDATES; + /* Reference (SimdPaddleOCR) defaults, paired with the Clipper-style + * round-join unclip and the scanline PolygonScore in the postprocess. */ + options->bitmap_threshold = 0.2f; + options->box_threshold = 0.45f; + options->unclip_ratio = 1.4f; + options->max_model_file_size = model_options.max_file_size; + options->max_workspace_size = session_options.max_workspace_size; + options->max_tensor_size = session_options.max_tensor_size; + options->max_image_pixels = LW_DET_DEFAULT_MAX_IMAGE_PIXELS; +} + +void lw_detector_info_init(lw_detector_info* info) { + if (info == NULL) + return; + memset(info, 0, sizeof(*info)); + info->struct_size = (uint32_t)sizeof(*info); +} + +void lw_detection_result_init(lw_detection_result* result) { + if (result == NULL) + return; + clear_result(result); +} + +static lw_status validate_options(const lw_detector_options* options, lw_detector_info* info, + lw_model_options* model_options, + lw_session_options* session_options, lw_error* error) { + lw_detector_options values; + lw_detector_options_init(&values); + if (options != NULL) { + if (!lw_abi_copy_input_prefix(&values, sizeof(values), options)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid detector options structure"); + return LW_STATUS_INVALID_ARGUMENT; + } + values.struct_size = (uint32_t)sizeof(values); + if (values.reserved != 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid detector options structure"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (values.limit_side_length == 0u) + values.limit_side_length = LW_DET_DEFAULT_LIMIT_SIDE_LENGTH; + if (values.max_candidates == 0u) + values.max_candidates = LW_DET_DEFAULT_MAX_CANDIDATES; + if (values.max_image_pixels == 0u) + values.max_image_pixels = LW_DET_DEFAULT_MAX_IMAGE_PIXELS; + } + if (values.limit_side_length < 32u || values.limit_side_length > LW_DET_MAX_LIMIT_SIDE_LENGTH || + values.max_candidates == 0u || values.max_candidates > LW_DET_MAX_CANDIDATES || + values.use_dilation > 1u || !isfinite(values.bitmap_threshold) || + values.bitmap_threshold < 0.0f || values.bitmap_threshold > 1.0f || + !isfinite(values.box_threshold) || values.box_threshold < 0.0f || + values.box_threshold > 1.0f || !isfinite(values.unclip_ratio) || + values.unclip_ratio <= 0.0f || values.unclip_ratio > 10.0f) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "detector thresholds or limits are invalid"); + return LW_STATUS_INVALID_ARGUMENT; + } + lw_model_options_init(model_options); + lw_session_options_init(session_options); + if (values.max_model_file_size != 0u) + model_options->max_file_size = values.max_model_file_size; + if (values.max_workspace_size != 0u) + session_options->max_workspace_size = values.max_workspace_size; + if (values.max_tensor_size != 0u) + session_options->max_tensor_size = values.max_tensor_size; + lw_detector_info_init(info); + info->limit_side_length = values.limit_side_length; + info->max_candidates = values.max_candidates; + info->use_dilation = values.use_dilation; + info->bitmap_threshold = values.bitmap_threshold; + info->box_threshold = values.box_threshold; + info->unclip_ratio = values.unclip_ratio; + info->max_image_pixels = values.max_image_pixels; + return LW_STATUS_OK; +} + +/* Tiny-family sniff: the reference treats a detector whose first Conv emits + * at most 16 channels as tiny and lowers its box threshold to 0.4. */ +static int model_is_tiny(const lw_model* model) { + uint32_t node_count; + uint32_t i; + if (model == NULL || model->node_offset == 0u || model->tensor_offset == 0u) return 0; + node_count = model->info.node_count; + for (i = 0u; i < node_count; ++i) { + const uint8_t* node = model->bytes + (size_t)model->node_offset + + (size_t)i * LWM_V0_NODE_SIZE; + uint32_t output_index; + const uint8_t* tensor; + if (lwm_read_u16(node) != LW_OP_CONV) continue; + output_index = lwm_read_u32(node + 40u); + if (output_index >= model->info.tensor_count) return 0; + tensor = model->bytes + (size_t)model->tensor_offset + + (size_t)output_index * LWM_V0_TENSOR_SIZE; + return lwm_read_i32(tensor + 8u + 4u) <= 16; + } + return 0; +} + +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +/* Compile and instantiate the x64 DET backend for the current input size. + * NHWC arm preferred (the promoted winner), NCHW fallback; any failure + * leaves the fields NULL, which keeps the canonical path. Enabled by default + * on x64 AVX2+FMA builds; the test disable hook reverts to canonical. */ +static void detector_try_backend(lw_detector* detector, uint32_t width, uint32_t height) { + lw_error backend_error; + lw_x64_det_program* program = NULL; + lw_x64_det_instance* instance = NULL; + lw_error_init(&backend_error); + if (detector->x64_backend_enabled == 0u) { + program = NULL; + instance = NULL; + } else if (lw_x64_det_backend_compile_ex(detector->model, height, width, + LW_X64_DET_COMPILE_NHWC, &program, + &backend_error) != LW_X64_DET_COMPILE_OK || + lw_x64_det_instance_create(program, &instance, &backend_error) != LW_STATUS_OK) { + lw_x64_det_program_free(program); + program = NULL; + instance = NULL; +#if !defined(__EMSCRIPTEN__) + if (lw_x64_det_backend_compile_ex(detector->model, height, width, + LW_X64_DET_COMPILE_NCHW, &program, + &backend_error) == LW_X64_DET_COMPILE_OK && + lw_x64_det_instance_create(program, &instance, &backend_error) != LW_STATUS_OK) { + lw_x64_det_program_free(program); + program = NULL; + instance = NULL; + } +#endif + } + lw_x64_det_program_free(detector->x64_program); + lw_x64_det_instance_free(detector->x64_instance); + detector->x64_program = program; + detector->x64_instance = instance; +#if defined(LW_WASM_COMPILED_DET) + if (program != NULL && instance != NULL) { + uint64_t packed_bytes = 0u; + uint32_t constant_index; + for (constant_index = 0u; constant_index < program->packed_constant_count; + ++constant_index) { + packed_bytes += program->constants[constant_index].bytes; + } + (void)fprintf(stderr, + "LW_WASM_COMPILED_DET input=%ux%u layout=nhwc ops=%u unsupported=%u " + "conversions=%u direct_input=%u arena_bytes=%llu packed_bytes=%llu\n", + width, height, program->op_count, program->unsupported_nodes, + program->layout_conversions, program->direct_nhwc, + (unsigned long long)program->arena_bytes, + (unsigned long long)packed_bytes); + } +#endif +} + +void lw_detector_test_enable_x64_backend(lw_detector* detector) { + if (detector == NULL) { + return; + } + detector->x64_backend_enabled = 1u; + detector_try_backend(detector, detector->resized_width, detector->resized_height); +} + +void lw_detector_test_disable_x64_backend(lw_detector* detector) { + if (detector == NULL) { + return; + } + detector->x64_backend_enabled = 0u; + lw_x64_det_program_free(detector->x64_program); + detector->x64_program = NULL; + lw_x64_det_instance_free(detector->x64_instance); + detector->x64_instance = NULL; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) + /* The compiled WASM path keeps a metadata-only session. Recreate a full + * canonical session on the next detection after the test hook disables it. */ + lw_session_free(detector->session); + detector->session = NULL; +#endif +} +#endif + +static lw_status ensure_session(lw_detector* detector, uint32_t width, uint32_t height, + lw_error* error) { + lw_tensor_desc input_desc; + lw_tensor_desc output_desc; + lw_session* new_session = NULL; + float* new_input = NULL; + float* new_probabilities = NULL; + uint64_t plane; + uint64_t input_count; + lw_status status; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) + uint8_t needs_input = 1u; +#endif + if (detector->session != NULL && detector->resized_width == width && + detector->resized_height == height) { + return LW_STATUS_OK; + } + plane = (uint64_t)width * height; + if (plane > UINT64_MAX / 3u || plane > SIZE_MAX / sizeof(float)) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "detector tensor size overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + input_count = plane * 3u; + if (input_count > SIZE_MAX / sizeof(float)) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "detector input tensor size overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + lw_tensor_desc_init(&input_desc); + input_desc.dtype = LW_DTYPE_F32; + input_desc.rank = 4u; + input_desc.dimensions[0] = 1; + input_desc.dimensions[1] = 3; + input_desc.dimensions[2] = (int32_t)height; + input_desc.dimensions[3] = (int32_t)width; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) + status = detector->x64_backend_enabled != 0u + ? lw_session_create_metadata_only(detector->model, &input_desc, 1u, + &detector->session_options, &new_session, error) + : lw_session_create(detector->model, &input_desc, 1u, + &detector->session_options, &new_session, error); +#else + status = lw_session_create(detector->model, &input_desc, 1u, &detector->session_options, + &new_session, error); +#endif + if (status != LW_STATUS_OK) + goto fail; + lw_tensor_desc_init(&output_desc); + status = lw_session_get_output_desc(new_session, 0u, &output_desc); + if (status != LW_STATUS_OK || output_desc.dtype != LW_DTYPE_F32 || output_desc.rank != 4u || + output_desc.dimensions[0] != 1 || output_desc.dimensions[1] != 1 || + output_desc.dimensions[2] != (int32_t)height || + output_desc.dimensions[3] != (int32_t)width) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "detector model output must have shape [1,1,height,width]"); + status = LW_STATUS_INVALID_SHAPE; + goto fail; + } +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) + if (detector->x64_backend_enabled != 0u) { + detector_try_backend(detector, width, height); + if (detector->x64_instance == NULL) { + /* Preserve the canonical fallback if this shape cannot compile. */ + lw_session_free(new_session); + new_session = NULL; + status = lw_session_create(detector->model, &input_desc, 1u, + &detector->session_options, &new_session, error); + if (status != LW_STATUS_OK) goto fail; + } else if (detector->x64_program->direct_nhwc != 0u) { + needs_input = 0u; + } + } + if (needs_input != 0u) +#endif + new_input = (float*)malloc((size_t)input_count * sizeof(*new_input)); + new_probabilities = (float*)malloc((size_t)plane * sizeof(*new_probabilities)); + if ( +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) + (needs_input != 0u && new_input == NULL) || +#else + new_input == NULL || +#endif + new_probabilities == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate detector tensor buffers"); + status = LW_STATUS_OUT_OF_MEMORY; + goto fail; + } + free(detector->probabilities); + free(detector->input); + lw_session_free(detector->session); + detector->session = new_session; + lw_session_set_intra_op_thread_count(new_session, detector->intra_op_thread_count); + detector->input = new_input; + detector->probabilities = new_probabilities; + detector->input_element_count = input_count; + detector->probability_element_count = plane; + detector->resized_width = width; + detector->resized_height = height; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) && \ + !(defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET)) + detector_try_backend(detector, width, height); +#endif + return LW_STATUS_OK; + +fail: + free(new_probabilities); + free(new_input); + lw_session_free(new_session); + return status; +} + +lw_status lw_detector_create(const char* model_path_utf8, const lw_detector_options* options, + lw_detector** out_detector, lw_error* error) { + lw_model_options model_options; + lw_detector* detector = NULL; + lw_status status; + if (out_detector != NULL) + *out_detector = NULL; + if (model_path_utf8 == NULL || model_path_utf8[0] == '\0' || out_detector == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "model path and output detector are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + detector = (lw_detector*)calloc(1u, sizeof(*detector)); + if (detector == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate detector handle"); + return LW_STATUS_OUT_OF_MEMORY; + } + detector->intra_op_thread_count = 1u; + detector->reading_order = LW_READING_ORDER_HORIZONTAL_LTR; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + /* Promoted default: the sharded NHWC backend is on; failures inside + * detector_try_backend fall back to the canonical executor. */ +#if defined(__EMSCRIPTEN__) && !defined(LW_WASM_COMPILED_DET) + detector->x64_backend_enabled = 0u; +#else + detector->x64_backend_enabled = 1u; +#endif +#endif + status = validate_options(options, &detector->info, &model_options, &detector->session_options, + error); + if (status != LW_STATUS_OK) + goto fail; + status = lw_model_load(model_path_utf8, &model_options, &detector->model, error); + if (status != LW_STATUS_OK) + goto fail; + if (detector->info.box_threshold == 0.0f || + (detector->info.box_threshold == 0.45f && model_is_tiny(detector->model))) { + /* Auto box threshold (reference behavior): tiny detectors use 0.4, + * everything else 0.45. A caller explicitly setting 0.45 on a tiny + * model is indistinguishable from the default and also gets 0.4. */ + detector->info.box_threshold = model_is_tiny(detector->model) ? 0.4f : 0.45f; + } + status = ensure_session(detector, 32u, 32u, error); + if (status != LW_STATUS_OK) + goto fail; + *out_detector = detector; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + +fail: + lw_detector_free(detector); + return status; +} + +void lw_detector_set_intra_op_thread_count(lw_detector* detector, uint32_t thread_count) { + if (detector == NULL) { + return; + } + detector->intra_op_thread_count = thread_count == 0u ? 1u : thread_count; + lw_session_set_intra_op_thread_count(detector->session, detector->intra_op_thread_count); +} + +uint32_t lw_detector_get_intra_op_thread_count(const lw_detector* detector) { + return detector == NULL || detector->session == NULL ? 1u + : detector->session->intra_op_thread_count; +} + +void lw_detector_free(lw_detector* detector) { + if (detector == NULL) + return; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_det_program_free(detector->x64_program); + lw_x64_det_instance_free(detector->x64_instance); +#endif + free(detector->probabilities); + free(detector->input); + lw_session_free(detector->session); + lw_model_free(detector->model); + lw_db_postprocess_workspace_free(&detector->postprocess_workspace); + lw_det_preprocess_workspace_free(&detector->preprocess_workspace); + free(detector->padded_source); + free(detector); +} + +lw_status lw_detector_get_info(const lw_detector* detector, lw_detector_info* info) { + if (detector == NULL || info == NULL || + !lw_abi_copy_output_prefix(info, info->struct_size, &detector->info, + sizeof(detector->info))) + return LW_STATUS_INVALID_ARGUMENT; + return LW_STATUS_OK; +} + +lw_status lw_detector_set_reading_order(lw_detector* detector, uint32_t reading_order, + lw_error* error) { + if (detector == NULL || !lw_reading_order_is_valid(reading_order)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid reading order"); + return LW_STATUS_INVALID_ARGUMENT; + } + detector->reading_order = reading_order; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +lw_status lw_detector_get_reading_order(const lw_detector* detector, uint32_t* reading_order, + lw_error* error) { + if (detector == NULL || reading_order == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "detector and output are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + *reading_order = detector->reading_order; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +static lw_status detector_detect_bgr_u8_impl( + lw_detector* detector, const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, lw_detection_box* boxes, uint32_t box_capacity, + lw_detection_result* result, lw_pipeline_component_profile* profile, lw_error* error) { + uint64_t source_pixels; + uint32_t resized_width; + uint32_t resized_height; + uint32_t box_count = 0u; + float width_ratio; + float height_ratio; + lw_detection_result output; + uint32_t result_size; + uint64_t started; + lw_status status; +#if defined(LW_WASM_COMPILED_DET) + uint8_t direct_nhwc_input = 0u; +#endif + if (detector == NULL || source == NULL || result == NULL || + result->struct_size < sizeof(uint32_t) || (boxes == NULL && box_capacity != 0u)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "detector, BGR source, initialized result, and valid box buffer are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + result_size = result->struct_size; + clear_result(&output); + if (source_width == 0u || source_height == 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "source image dimensions must be positive"); + return LW_STATUS_INVALID_ARGUMENT; + } + source_pixels = (uint64_t)source_width * source_height; + if (source_pixels > detector->info.max_image_pixels) { + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, "source image exceeds max_image_pixels"); + return LW_STATUS_MEMORY_LIMIT; + } + started = lw_pipeline_profile_now(profile); + status = lw_det_compute_size(source_width, source_height, detector->info.limit_side_length, + &resized_width, &resized_height, &width_ratio, &height_ratio); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "unable to compute detector input size"); + return status; + } + output.resized_width = resized_width; + output.resized_height = resized_height; + output.width_ratio = width_ratio; + output.height_ratio = height_ratio; + status = ensure_session(detector, resized_width, resized_height, error); + if (status != LW_STATUS_OK) { + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return status; + } +#if defined(LW_WASM_COMPILED_DET) + direct_nhwc_input = detector->x64_instance != NULL && + detector->x64_program != NULL && detector->x64_program->direct_nhwc != 0u; +#endif +#if defined(LW_EXPERIMENTAL_DET_FIXED_POINT_RESIZE) + { + const uint8_t* preprocess_source = source; + uint32_t preprocess_width = source_width; + uint32_t preprocess_height = source_height; + uint32_t preprocess_stride = source_stride; + uint64_t preprocess_byte_count = source_byte_count; + /* Tiny-image pad: sources with w+h < 64 are zero-padded to a 32x32 + * buffer before resize (reference behavior). The ratios stay based on + * the original dimensions, so the postprocess mapping is unaffected. */ + if (source_width + source_height < 64u) { + uint32_t padded_width = source_width > 32u ? source_width : 32u; + uint32_t padded_height = source_height > 32u ? source_height : 32u; + uint64_t padded_bytes = (uint64_t)padded_width * padded_height * 3u; + uint32_t y; + if (detector->padded_width != padded_width || + detector->padded_height != padded_height) { + uint8_t* padded = (uint8_t*)realloc(detector->padded_source, (size_t)padded_bytes); + if (padded == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, + "tiny-image pad allocation failed"); + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return LW_STATUS_OUT_OF_MEMORY; + } + detector->padded_source = padded; + detector->padded_width = padded_width; + detector->padded_height = padded_height; + } + memset(detector->padded_source, 0, (size_t)padded_bytes); + for (y = 0u; y < source_height; ++y) { + memcpy(detector->padded_source + (size_t)y * padded_width * 3u, + source + (size_t)y * source_stride, (size_t)source_width * 3u); + } + preprocess_source = detector->padded_source; + preprocess_width = padded_width; + preprocess_height = padded_height; + preprocess_stride = padded_width * 3u; + preprocess_byte_count = padded_bytes; + } +#if defined(LW_WASM_COMPILED_DET) + if (direct_nhwc_input != 0u) { + uint64_t backend_input_count = 0u; + float* backend_input = lw_x64_det_instance_input(detector->x64_instance, + &backend_input_count); + if (backend_input == NULL || backend_input_count != detector->input_element_count) { + status = LW_STATUS_INVALID_ARGUMENT; + } else { + status = lw_det_preprocess_bgr_u8_fixed_nhwc( + preprocess_source, preprocess_byte_count, preprocess_width, + preprocess_height, preprocess_stride, resized_width, resized_height, + backend_input, backend_input_count, &detector->preprocess_workspace, + detector->session->thread_pool, detector->intra_op_thread_count); + } + } else +#endif + status = lw_det_preprocess_bgr_u8_fixed( + preprocess_source, preprocess_byte_count, preprocess_width, preprocess_height, + preprocess_stride, resized_width, resized_height, detector->input, + detector->input_element_count, &detector->preprocess_workspace, + detector->session->thread_pool, detector->intra_op_thread_count); + } +#else + status = lw_det_preprocess_bgr_u8(source, source_byte_count, source_width, source_height, + source_stride, resized_width, resized_height, detector->input, + detector->input_element_count); +#endif + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "BGR source layout is invalid"); + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return status; + } +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) + if (lw_profile_env_is_one("LW_WASM_DET_DEBUG")) { + if (direct_nhwc_input != 0u) { + uint64_t backend_input_count = 0u; + const float* backend_input = lw_x64_det_instance_input( + detector->x64_instance, &backend_input_count); + if (backend_input != NULL) + detector_debug_tensor("input_nhwc", backend_input, backend_input_count); + } else { + detector_debug_tensor("input_nchw", detector->input, + detector->input_element_count); + } + } +#endif + lw_pipeline_profile_add_elapsed(profile == NULL ? NULL : &profile->preprocess_nanoseconds, + started, profile); + started = lw_pipeline_profile_now(profile); +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + if (detector->x64_instance != NULL) { + uint64_t backend_input_count = 0u; + float* backend_input = lw_x64_det_instance_input(detector->x64_instance, + &backend_input_count); + if (backend_input == NULL || backend_input_count != detector->input_element_count) { + status = LW_STATUS_INVALID_ARGUMENT; + } else { +#if defined(LW_WASM_COMPILED_DET) + if (direct_nhwc_input == 0u) +#endif + memcpy(backend_input, detector->input, + (size_t)backend_input_count * sizeof(float)); + status = lw_x64_det_instance_run_profiled_ex( + detector->x64_instance, detector->probabilities, + detector->probability_element_count, detector->session->thread_pool, + detector->intra_op_thread_count, + profile == NULL ? NULL : &profile->execution, error); + if (status == LW_STATUS_OK && profile != NULL) + lw_profile_add_value(&profile->compiled_backend_runs, 1u); + } + } else +#endif + { + status = profile == NULL + ? lw_execute_session_f32(detector->session, detector->input, + detector->input_element_count, + detector->probabilities, + detector->probability_element_count, error) + : lw_execute_session_f32_profiled( + detector->session, detector->input, detector->input_element_count, + detector->probabilities, detector->probability_element_count, + &profile->execution, error); + if (status == LW_STATUS_OK && profile != NULL) + lw_profile_add_value(&profile->canonical_fallback_runs, 1u); + } + lw_pipeline_profile_add_elapsed(profile == NULL ? NULL : &profile->graph_nanoseconds, started, + profile); + if (status != LW_STATUS_OK) { + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return status; + } +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) + if (lw_profile_env_is_one("LW_WASM_DET_DEBUG")) + detector_debug_tensor("probabilities", detector->probabilities, + detector->probability_element_count); +#endif + started = lw_pipeline_profile_now(profile); + status = lw_db_postprocess_f32_ws(detector->probabilities, resized_width, resized_height, + detector->info.bitmap_threshold, detector->info.box_threshold, + detector->info.unclip_ratio, detector->info.use_dilation, + detector->info.max_candidates, source_width, source_height, + width_ratio, height_ratio, boxes, box_capacity, &box_count, + &detector->postprocess_workspace, profile); + lw_pipeline_profile_add_elapsed(profile == NULL ? NULL : &profile->postprocess_nanoseconds, + started, profile); + output.box_count = box_count; + output.required_box_capacity = box_count; + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, + status == LW_STATUS_OUT_OF_BOUNDS ? "box buffer capacity is insufficient" + : "detector postprocessing failed"); + return status; + } + if (boxes != NULL) { + status = lw_sort_detection_boxes(boxes, box_count, detector->reading_order); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "unable to sort detection boxes"); + return status; + } + } + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +lw_status lw_detector_detect_bgr_u8(lw_detector* detector, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + lw_detection_box* boxes, uint32_t box_capacity, + lw_detection_result* result, lw_error* error) { + return detector_detect_bgr_u8_impl(detector, source, source_byte_count, source_width, + source_height, source_stride, boxes, box_capacity, result, + NULL, error); +} + +lw_status lw_detector_detect_bgr_u8_profiled( + lw_detector* detector, const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, lw_detection_box* boxes, uint32_t box_capacity, + lw_detection_result* result, lw_pipeline_component_profile* profile, lw_error* error) { + if (profile == NULL || profile->execution.struct_size != sizeof(profile->execution) || + profile->execution.clock == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "an initialized detector profile and clock are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + return detector_detect_bgr_u8_impl(detector, source, source_byte_count, source_width, + source_height, source_stride, boxes, box_capacity, result, + profile, error); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/ocr.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/ocr.c new file mode 100644 index 00000000..ef30662e --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/ocr.c @@ -0,0 +1,1309 @@ +#include "lw_infer.h" +#include "abi_compat_internal.h" + +/* + * Full OCR composition layer: DET -> perspective crop -> optional CLS -> REC. + * Reusable scratch buffers belong to the handle, so one handle must not be + * entered concurrently. Applications can create multiple handles for parallel + * requests without adding locks inside the dependency-free core. + */ + +#include "crop_buffer_internal.h" +#include "crop_internal.h" +#include "cpu_topology.h" +#include "det_internal.h" +#include "error_internal.h" +#include "ocr_internal.h" +#include "parallel_internal.h" +#include "profile_internal.h" +#include "rec_internal.h" + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# define WIN32_LEAN_AND_MEAN +# include +# include +#elif !defined(__EMSCRIPTEN__) +# include +# include +#endif + +#define LW_OCR_DEFAULT_MAX_CROP_PIXELS UINT64_C(16000000) +#define LW_OCR_DEFAULT_RETAINED_CROP_BYTES (UINT64_C(4) * 1024u * 1024u) +#define LW_OCR_MAX_WORKER_COUNT 16u + +typedef struct lw_ocr_crop { + uint64_t offset; + uint64_t byte_count; + uint32_t width; + uint32_t height; + uint32_t box_index; + uint32_t target_width; + uint8_t scheduled; +} lw_ocr_crop; + +/* Row-band job for the parallel crop phase: bands are row-disjoint output + * ranges with identical per-pixel math, so the phase is bit-identical to the + * serial crop loop. */ +typedef struct lw_ocr_crop_band { + uint32_t crop_index; + uint32_t row_begin; + uint32_t row_end; +} lw_ocr_crop_band; + +typedef struct lw_ocr_worker_task { + lw_ocr* ocr; + uint32_t worker_index; + uint32_t initial_crop_slot; + lw_status status; + lw_error error; + uint32_t profile_enabled; + uint64_t wall_nanoseconds; + uint64_t rec_width_sample_count; + uint64_t rec_resized_width_sum; + uint64_t rec_target_width_sum; + uint64_t rec_width_histogram[LW_REC_WIDTH_HISTOGRAM_BUCKET_COUNT]; + lw_pipeline_component_profile classifier_profile; + lw_pipeline_component_profile recognizer_profile; + lw_crop_buffer crop_buffer; +#if defined(LW_EXPERIMENTAL_STREAMING_CROPS) + const uint8_t* source; + uint64_t source_byte_count; + uint32_t source_width; + uint32_t source_height; + uint32_t source_stride; + uint64_t crop_nanoseconds; +#endif +} lw_ocr_worker_task; + +struct lw_ocr { + lw_detector* detector; + lw_classifier** classifiers; + lw_recognizer** recognizers; + uint32_t worker_count; + uint32_t det_intra_op_thread_cap; + lw_cpu_topology cpu_topology; + lw_detection_box* detected_boxes; + lw_ocr_line* scratch_lines; + char* scratch_text; + lw_ocr_crop* crops; + uint32_t* crop_schedule; + uint32_t crop_schedule_count; +#if defined(_WIN32) + volatile LONG next_crop_slot; +#else + uint32_t next_crop_slot; +#endif + lw_ocr_worker_task* worker_tasks; +#if defined(_WIN32) + HANDLE* worker_threads; +#elif !defined(__EMSCRIPTEN__) + pthread_t* worker_threads; +#endif + uint8_t* worker_started; + uint8_t* crop; + uint64_t crop_capacity; + uint64_t max_crop_pixels; + uint64_t text_capacity_per_line; + float classifier_threshold; + lw_thread_pool* line_pool; + lw_ocr_crop_band* crop_bands; + uint32_t crop_band_capacity; + uint32_t crop_band_count; + volatile lw_status crop_band_status; + lw_ocr_info info; +}; + +/* Use a width-neutral allocation check so GCC does not reject valid 64-bit + * builds when a bounded uint32_t count is compared directly with SIZE_MAX. */ +static int allocation_fits(uint64_t count, size_t element_size) { + return element_size != 0u && count <= (uint64_t)(SIZE_MAX / element_size); +} + +static uint32_t default_worker_count(void) { + lw_cpu_topology topology; + lw_cpu_topology_detect(&topology); + return lw_parallel_default_line_worker_count(&topology); +} + +static void clear_result(lw_ocr_result* result) { + memset(result, 0, sizeof(*result)); + result->struct_size = (uint32_t)sizeof(*result); +} + +void lw_ocr_options_init(lw_ocr_options* options) { + if (options == NULL) + return; + memset(options, 0, sizeof(*options)); + options->struct_size = (uint32_t)sizeof(*options); + options->use_direction_classification = 1u; + options->classifier_threshold = 0.9f; + options->worker_count = default_worker_count(); + options->max_crop_pixels = LW_OCR_DEFAULT_MAX_CROP_PIXELS; + lw_detector_options_init(&options->detector); + lw_classifier_options_init(&options->classifier); + lw_recognizer_options_init(&options->recognizer); +} + +void lw_ocr_info_init(lw_ocr_info* info) { + if (info == NULL) + return; + memset(info, 0, sizeof(*info)); + info->struct_size = (uint32_t)sizeof(*info); +} + +void lw_ocr_result_init(lw_ocr_result* result) { + if (result == NULL) + return; + clear_result(result); +} + +static lw_status validate_options(const lw_ocr_options* options, lw_ocr_options* values, + lw_error* error) { + lw_ocr_options raw; + size_t supplied_size; + size_t fixed_size = offsetof(lw_ocr_options, detector); + lw_ocr_options_init(values); + if (options != NULL) { + uint32_t declared_size; + size_t detector_bytes; + size_t classifier_bytes; + size_t recognizer_bytes; + lw_detector_options detector_defaults; + lw_classifier_options classifier_defaults; + lw_recognizer_options recognizer_defaults; + + memcpy(&declared_size, options, sizeof(declared_size)); + if (declared_size < sizeof(uint32_t)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid OCR options structure"); + return LW_STATUS_INVALID_ARGUMENT; + } + supplied_size = (size_t)declared_size; + if (supplied_size > sizeof(raw)) { + supplied_size = sizeof(raw); + } + memset(&raw, 0, sizeof(raw)); + memcpy(&raw, options, supplied_size); + if (supplied_size < fixed_size) { + fixed_size = supplied_size; + } + memcpy(values, &raw, fixed_size); + + lw_detector_options_init(&detector_defaults); + detector_bytes = supplied_size > offsetof(lw_ocr_options, detector) + ? supplied_size - offsetof(lw_ocr_options, detector) + : 0u; + if (!lw_abi_copy_nested_input_prefix(&detector_defaults, sizeof(detector_defaults), + &raw.detector, detector_bytes)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid OCR detector options"); + return LW_STATUS_INVALID_ARGUMENT; + } + detector_defaults.struct_size = (uint32_t)sizeof(detector_defaults); + values->detector = detector_defaults; + + lw_classifier_options_init(&classifier_defaults); + classifier_bytes = supplied_size > offsetof(lw_ocr_options, classifier) + ? supplied_size - offsetof(lw_ocr_options, classifier) + : 0u; + if (!lw_abi_copy_nested_input_prefix(&classifier_defaults, sizeof(classifier_defaults), + &raw.classifier, classifier_bytes)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid OCR classifier options"); + return LW_STATUS_INVALID_ARGUMENT; + } + classifier_defaults.struct_size = (uint32_t)sizeof(classifier_defaults); + values->classifier = classifier_defaults; + + lw_recognizer_options_init(&recognizer_defaults); + recognizer_bytes = supplied_size > offsetof(lw_ocr_options, recognizer) + ? supplied_size - offsetof(lw_ocr_options, recognizer) + : 0u; + if (!lw_abi_copy_nested_input_prefix(&recognizer_defaults, sizeof(recognizer_defaults), + &raw.recognizer, recognizer_bytes)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid OCR recognizer options"); + return LW_STATUS_INVALID_ARGUMENT; + } + recognizer_defaults.struct_size = (uint32_t)sizeof(recognizer_defaults); + values->recognizer = recognizer_defaults; + values->struct_size = (uint32_t)sizeof(*values); + if (values->max_crop_pixels == 0u) + values->max_crop_pixels = LW_OCR_DEFAULT_MAX_CROP_PIXELS; + } + if (values->use_direction_classification > 1u || !isfinite(values->classifier_threshold) || + values->classifier_threshold < 0.0f || values->classifier_threshold > 1.0f || + values->detector.struct_size != sizeof(values->detector) || + values->classifier.struct_size != sizeof(values->classifier) || + values->recognizer.struct_size != sizeof(values->recognizer) || + values->worker_count > LW_OCR_MAX_WORKER_COUNT || values->detector.reserved != 0u || + values->classifier.reserved != 0u || values->recognizer.reserved0 != 0u || + values->recognizer.reserved1 != 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "OCR classifier policy or nested options are invalid"); + return LW_STATUS_INVALID_ARGUMENT; + } + return LW_STATUS_OK; +} + +lw_status lw_ocr_create(const char* detector_model_path_utf8, + const char* classifier_model_path_utf8, + const char* recognizer_model_path_utf8, const char* dictionary_path_utf8, + const lw_ocr_options* options, lw_ocr** out_ocr, lw_error* error) { + lw_ocr_options values; + lw_detector_info detector_info; + lw_recognizer_info recognizer_info; + lw_ocr* ocr = NULL; + uint64_t maximum_text_capacity; + lw_status status; + if (out_ocr != NULL) + *out_ocr = NULL; + if (detector_model_path_utf8 == NULL || detector_model_path_utf8[0] == '\0' || + recognizer_model_path_utf8 == NULL || recognizer_model_path_utf8[0] == '\0' || + dictionary_path_utf8 == NULL || dictionary_path_utf8[0] == '\0' || out_ocr == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "DET model, REC model, dictionary, and output OCR handle are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + status = validate_options(options, &values, error); + if (status != LW_STATUS_OK) + return status; + if (values.worker_count == 0u) + values.worker_count = default_worker_count(); +#if defined(__EMSCRIPTEN__) + if (values.worker_count != 1u) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "WebAssembly OCR requires worker_count=1 without pthreads"); + return LW_STATUS_UNSUPPORTED; + } +#endif + if (values.use_direction_classification != 0u && + (classifier_model_path_utf8 == NULL || classifier_model_path_utf8[0] == '\0')) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "CLS model is required when direction classification is enabled"); + return LW_STATUS_INVALID_ARGUMENT; + } + ocr = (lw_ocr*)calloc(1u, sizeof(*ocr)); + if (ocr == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate OCR handle"); + return LW_STATUS_OUT_OF_MEMORY; + } + status = lw_detector_create(detector_model_path_utf8, &values.detector, &ocr->detector, error); + if (status != LW_STATUS_OK) + goto fail; + lw_cpu_topology_detect(&ocr->cpu_topology); + ocr->det_intra_op_thread_cap = lw_parallel_default_det_thread_count(&ocr->cpu_topology); + lw_detector_set_intra_op_thread_count(ocr->detector, ocr->det_intra_op_thread_cap); + ocr->worker_count = values.worker_count; + ocr->classifiers = (lw_classifier**)calloc(ocr->worker_count, sizeof(*ocr->classifiers)); + ocr->recognizers = (lw_recognizer**)calloc(ocr->worker_count, sizeof(*ocr->recognizers)); + ocr->worker_tasks = (lw_ocr_worker_task*)calloc(ocr->worker_count, sizeof(*ocr->worker_tasks)); + ocr->worker_started = (uint8_t*)calloc(ocr->worker_count, sizeof(*ocr->worker_started)); +#if defined(_WIN32) || !defined(__EMSCRIPTEN__) + ocr->worker_threads = calloc(ocr->worker_count, sizeof(*ocr->worker_threads)); +#endif + if (ocr->classifiers == NULL || ocr->recognizers == NULL || ocr->worker_tasks == NULL || + ocr->worker_started == NULL +#if defined(_WIN32) || !defined(__EMSCRIPTEN__) + || ocr->worker_threads == NULL +#endif + ) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate OCR worker pool"); + status = LW_STATUS_OUT_OF_MEMORY; + goto fail; + } + { + uint32_t worker_index; + for (worker_index = 0u; worker_index < ocr->worker_count; ++worker_index) { + lw_crop_buffer_init(&ocr->worker_tasks[worker_index].crop_buffer, + LW_OCR_DEFAULT_RETAINED_CROP_BYTES); + if (values.use_direction_classification != 0u) { + status = lw_classifier_create(classifier_model_path_utf8, &values.classifier, + &ocr->classifiers[worker_index], error); + if (status != LW_STATUS_OK) + goto fail; + } + if (worker_index == 0u) { + status = lw_recognizer_create(recognizer_model_path_utf8, dictionary_path_utf8, + &values.recognizer, &ocr->recognizers[worker_index], + error); + } else { + status = lw_recognizer_clone(ocr->recognizers[0], + &ocr->recognizers[worker_index], error); + } + if (status != LW_STATUS_OK) + goto fail; + { + /* CTC 头 matmul 扇出到 recognizer 自有池:每个 recognizer 同时只服务一行, + * 不会竞争;额度按行 worker 数均分。 */ + uint32_t rec_intra = ocr->det_intra_op_thread_cap; + if (ocr->worker_count > 1u) { + rec_intra = rec_intra / ocr->worker_count; + if (rec_intra == 0u) + rec_intra = 1u; + } + lw_recognizer_set_intra_op_thread_count(ocr->recognizers[worker_index], + rec_intra); + } + if (worker_index == 0u && values.recognizer.target_width > 320u) { + status = + lw_recognizer_enable_adaptive_width(ocr->recognizers[worker_index], 1u, error); + if (status != LW_STATUS_OK) + goto fail; + } +#if defined(LW_REC_RESIDENT_WIDTHS) && LW_REC_RESIDENT_WIDTHS + if (values.recognizer.target_width == 960u) { + status = lw_recognizer_enable_resident_widths(ocr->recognizers[worker_index], error); + if (status != LW_STATUS_OK) + goto fail; + } +#endif + } + } + lw_detector_info_init(&detector_info); + lw_recognizer_info_init(&recognizer_info); + if (lw_detector_get_info(ocr->detector, &detector_info) != LW_STATUS_OK || + lw_recognizer_get_info(ocr->recognizers[0], &recognizer_info) != LW_STATUS_OK || + detector_info.max_candidates == 0u || recognizer_info.max_text_capacity == 0u) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "unable to query OCR component capacities"); + status = LW_STATUS_INVALID_SHAPE; + goto fail; + } + if ((uint64_t)detector_info.max_candidates > UINT64_MAX / recognizer_info.max_text_capacity) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "OCR maximum text capacity overflows"); + status = LW_STATUS_OUT_OF_BOUNDS; + goto fail; + } + maximum_text_capacity = + (uint64_t)detector_info.max_candidates * recognizer_info.max_text_capacity; + if (!allocation_fits(detector_info.max_candidates, sizeof(*ocr->detected_boxes)) || + !allocation_fits(detector_info.max_candidates, sizeof(*ocr->scratch_lines)) || + maximum_text_capacity > SIZE_MAX) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, + "OCR output scratch capacity exceeds the platform"); + status = LW_STATUS_OUT_OF_BOUNDS; + goto fail; + } + ocr->detected_boxes = + (lw_detection_box*)calloc(detector_info.max_candidates, sizeof(*ocr->detected_boxes)); + ocr->scratch_lines = + (lw_ocr_line*)calloc(detector_info.max_candidates, sizeof(*ocr->scratch_lines)); + ocr->crops = (lw_ocr_crop*)calloc(detector_info.max_candidates, sizeof(*ocr->crops)); + ocr->crop_schedule = + (uint32_t*)malloc((size_t)detector_info.max_candidates * sizeof(*ocr->crop_schedule)); + ocr->scratch_text = (char*)malloc((size_t)maximum_text_capacity); + if (ocr->detected_boxes == NULL || ocr->scratch_lines == NULL || ocr->crops == NULL || + ocr->crop_schedule == NULL || ocr->scratch_text == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, + "unable to allocate OCR output scratch buffers"); + status = LW_STATUS_OUT_OF_MEMORY; + goto fail; + } + ocr->max_crop_pixels = values.max_crop_pixels; + ocr->text_capacity_per_line = recognizer_info.max_text_capacity; + ocr->classifier_threshold = values.classifier_threshold; + lw_ocr_info_init(&ocr->info); + ocr->info.use_direction_classification = values.use_direction_classification; + ocr->info.max_line_capacity = detector_info.max_candidates; + ocr->info.worker_count = ocr->worker_count; + ocr->info.max_text_capacity = maximum_text_capacity; + ocr->info.max_text_capacity_per_line = recognizer_info.max_text_capacity; + ocr->info.max_crop_pixels = values.max_crop_pixels; + *out_ocr = ocr; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + +fail: + lw_ocr_free(ocr); + return status; +} + +void lw_ocr_set_det_intra_op_thread_count(lw_ocr* ocr, uint32_t thread_count) { + if (ocr == NULL) { + return; + } + if (thread_count == 0u) { + thread_count = lw_parallel_default_det_thread_count(&ocr->cpu_topology); + } + if (thread_count > LW_OCR_MAX_WORKER_COUNT) { + thread_count = LW_OCR_MAX_WORKER_COUNT; + } + ocr->det_intra_op_thread_cap = thread_count; + lw_detector_set_intra_op_thread_count(ocr->detector, thread_count); +} + +uint32_t lw_ocr_get_det_intra_op_thread_count(const lw_ocr* ocr) { + return ocr == NULL ? 1u : lw_detector_get_intra_op_thread_count(ocr->detector); +} + +uint32_t lw_ocr_get_det_intra_op_thread_cap(const lw_ocr* ocr) { + return ocr == NULL ? 1u : ocr->det_intra_op_thread_cap; +} + +void lw_ocr_get_cpu_topology(const lw_ocr* ocr, lw_cpu_topology* topology) { + if (topology == NULL) { + return; + } + if (ocr == NULL) { + memset(topology, 0, sizeof(*topology)); + return; + } + *topology = ocr->cpu_topology; +} + +void lw_ocr_free(lw_ocr* ocr) { + uint32_t worker_index; + if (ocr == NULL) + return; + free(ocr->crop); + for (worker_index = 0u; worker_index < ocr->worker_count; ++worker_index) { + if (ocr->worker_tasks != NULL) { + lw_crop_buffer_free(&ocr->worker_tasks[worker_index].crop_buffer); + } + } + free(ocr->worker_started); +#if defined(_WIN32) || !defined(__EMSCRIPTEN__) + free(ocr->worker_threads); +#endif + lw_thread_pool_free(ocr->line_pool); + free(ocr->crop_bands); + free(ocr->worker_tasks); + free(ocr->crop_schedule); + free(ocr->crops); + free(ocr->scratch_text); + free(ocr->scratch_lines); + free(ocr->detected_boxes); + for (worker_index = 0u; worker_index < ocr->worker_count; ++worker_index) { + lw_recognizer_free(ocr->recognizers == NULL ? NULL : ocr->recognizers[worker_index]); + lw_classifier_free(ocr->classifiers == NULL ? NULL : ocr->classifiers[worker_index]); + } + free(ocr->recognizers); + free(ocr->classifiers); + lw_detector_free(ocr->detector); + free(ocr); +} + +lw_status lw_ocr_get_info(const lw_ocr* ocr, lw_ocr_info* info) { + if (ocr == NULL || info == NULL || + !lw_abi_copy_output_prefix(info, info->struct_size, &ocr->info, sizeof(ocr->info))) + return LW_STATUS_INVALID_ARGUMENT; + return LW_STATUS_OK; +} + +lw_status lw_ocr_set_reading_order(lw_ocr* ocr, uint32_t reading_order, lw_error* error) { + if (ocr == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "OCR handle is required"); + return LW_STATUS_INVALID_ARGUMENT; + } + return lw_detector_set_reading_order(ocr->detector, reading_order, error); +} + +lw_status lw_ocr_get_reading_order(const lw_ocr* ocr, uint32_t* reading_order, lw_error* error) { + if (ocr == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "OCR handle is required"); + return LW_STATUS_INVALID_ARGUMENT; + } + return lw_detector_get_reading_order(ocr->detector, reading_order, error); +} + +static lw_status ensure_crop_capacity(lw_ocr* ocr, uint64_t byte_count, lw_error* error) { + uint8_t* resized; + if (byte_count <= ocr->crop_capacity) + return LW_STATUS_OK; + if (byte_count > SIZE_MAX) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "OCR crop capacity exceeds the platform"); + return LW_STATUS_OUT_OF_BOUNDS; + } + resized = (uint8_t*)realloc(ocr->crop, (size_t)byte_count); + if (resized == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate OCR crop buffer"); + return LW_STATUS_OUT_OF_MEMORY; + } + ocr->crop = resized; + ocr->crop_capacity = byte_count; + return LW_STATUS_OK; +} + +/* Each worker owns its CLS and REC handles, while crop/output slots are + * disjoint. This keeps the public OCR handle non-reentrant but lets one request + * recognize independent text lines concurrently. */ +static void process_worker_task(lw_ocr_worker_task* task) { + lw_ocr* ocr = task->ocr; + task->status = LW_STATUS_OK; + lw_error_init(&task->error); + for (;;) { + uint32_t slot; + uint32_t index; + if (task->initial_crop_slot != UINT32_MAX) { + slot = task->initial_crop_slot; + task->initial_crop_slot = UINT32_MAX; + } else { +#if defined(_WIN32) + slot = (uint32_t)(InterlockedIncrement(&ocr->next_crop_slot) - 1); +#elif defined(__EMSCRIPTEN__) + slot = ocr->next_crop_slot++; +#else + slot = __atomic_fetch_add(&ocr->next_crop_slot, 1u, __ATOMIC_RELAXED); +#endif + } + if (slot >= ocr->crop_schedule_count) + return; + index = ocr->crop_schedule[slot]; + lw_ocr_crop* crop = &ocr->crops[index]; + lw_detection_box* box = &ocr->detected_boxes[crop->box_index]; + lw_classification_result classification; + lw_recognition_result recognition; + lw_ocr_line* line = &ocr->scratch_lines[index]; + uint8_t* crop_pixels; + uint32_t crop_width = crop->width; + uint32_t crop_height = crop->height; + uint64_t crop_byte_count = crop->byte_count; + uint64_t text_offset = (uint64_t)index * ocr->text_capacity_per_line; + char* line_text = ocr->scratch_text + (size_t)text_offset; + +#if defined(LW_EXPERIMENTAL_STREAMING_CROPS) + { + uint64_t crop_started = + task->profile_enabled == 0u ? 0u : lw_pipeline_profile_now(&task->recognizer_profile); + lw_status crop_status; + crop_status = lw_crop_buffer_reserve(&task->crop_buffer, crop_byte_count, &task->error); + if (crop_status == LW_STATUS_OK) { + crop_status = lw_crop_quad_bgr_u8( + task->source, task->source_byte_count, task->source_width, task->source_height, + task->source_stride, box, task->crop_buffer.data, task->crop_buffer.capacity, &crop_width, + &crop_height, &crop_byte_count); + } + if (task->profile_enabled != 0u) { + uint64_t crop_finished = lw_pipeline_profile_now(&task->recognizer_profile); + uint64_t elapsed = crop_finished >= crop_started ? crop_finished - crop_started : 0u; + if (task->crop_nanoseconds <= UINT64_MAX - elapsed) { + task->crop_nanoseconds += elapsed; + } else { + task->crop_nanoseconds = UINT64_MAX; + } + } + if (crop_status != LW_STATUS_OK) { + task->status = crop_status; + return; + } + crop_pixels = task->crop_buffer.data; + } +#else + crop_pixels = ocr->crop + (size_t)crop->offset; +#endif + + memset(&classification, 0, sizeof(classification)); + if (ocr->classifiers[task->worker_index] != NULL) { + lw_classification_result_init(&classification); + task->status = task->profile_enabled == 0u + ? lw_classifier_classify_bgr_u8( + ocr->classifiers[task->worker_index], crop_pixels, + crop_byte_count, crop_width, crop_height, crop_width * 3u, + &classification, &task->error) + : lw_classifier_classify_bgr_u8_profiled( + ocr->classifiers[task->worker_index], crop_pixels, + crop_byte_count, crop_width, crop_height, crop_width * 3u, + &classification, &task->classifier_profile, &task->error); + if (task->status != LW_STATUS_OK) + return; + if ((classification.label & 1u) != 0u && + classification.score > ocr->classifier_threshold) { + lw_rotate_bgr_u8_180(crop_pixels, crop_width, crop_height); + } + } + + lw_recognition_result_init(&recognition); + task->status = task->profile_enabled == 0u + ? lw_recognizer_recognize_bgr_u8( + ocr->recognizers[task->worker_index], crop_pixels, + crop_byte_count, crop_width, crop_height, crop_width * 3u, + line_text, ocr->text_capacity_per_line, &recognition, &task->error) + : lw_recognizer_recognize_bgr_u8_profiled( + ocr->recognizers[task->worker_index], crop_pixels, + crop_byte_count, crop_width, crop_height, crop_width * 3u, + line_text, ocr->text_capacity_per_line, &recognition, + &task->recognizer_profile, &task->error); + if (task->status != LW_STATUS_OK) + return; + if (recognition.required_text_capacity == 0u || + recognition.required_text_capacity > ocr->text_capacity_per_line) { + task->status = LW_STATUS_OUT_OF_BOUNDS; + lw_set_error(&task->error, task->status, + "recognizer returned an invalid text capacity"); + return; + } + if (task->profile_enabled != 0u) { + uint32_t bucket = lw_rec_width_histogram_bucket(recognition.resized_width); + lw_profile_add_value(&task->rec_width_sample_count, 1u); + lw_profile_add_value(&task->rec_resized_width_sum, recognition.resized_width); + lw_profile_add_value( + &task->rec_target_width_sum, + crop->target_width); + lw_profile_add_value(&task->rec_width_histogram[bucket], 1u); + } + + memset(line, 0, sizeof(*line)); + line->box = *box; + line->recognition_score = recognition.score; + line->classification_score = classification.score; + line->classification_label = classification.label; + line->applied_rotation_degrees = ocr->classifiers[task->worker_index] != NULL && + (classification.label & 1u) != 0u && + classification.score > ocr->classifier_threshold + ? 180u + : 0u; + line->emitted_count = recognition.emitted_count; + line->text_offset = text_offset; + line->text_length = recognition.required_text_capacity - 1u; + } +} + +static void execute_worker_task(lw_ocr_worker_task* task) { + uint64_t started = + task->profile_enabled == 0u ? 0u : lw_pipeline_profile_now(&task->recognizer_profile); + process_worker_task(task); + if (task->profile_enabled != 0u) { + uint64_t finished = lw_pipeline_profile_now(&task->recognizer_profile); + task->wall_nanoseconds = finished >= started ? finished - started : 0u; + } +} + +#if !defined(LW_EXPERIMENTAL_PERSISTENT_LINE_POOL) +#if defined(_WIN32) +static unsigned __stdcall worker_entry(void* context) { + execute_worker_task((lw_ocr_worker_task*)context); + return 0u; +} +#elif !defined(__EMSCRIPTEN__) +static void* worker_entry(void* context) { + execute_worker_task((lw_ocr_worker_task*)context); + return NULL; +} +#endif +#endif + +#if defined(LW_EXPERIMENTAL_PERSISTENT_LINE_POOL) +/* Pool callback for the persistent line workers; worker 0 runs on the + * caller thread inside lw_thread_pool_run, the rest on pool threads. */ +static void execute_line_worker(void* context, uint32_t worker_index, uint32_t worker_count) { + lw_ocr* ocr = (lw_ocr*)context; + (void)worker_count; + execute_worker_task(&ocr->worker_tasks[worker_index]); +} +#endif + +#if defined(LW_EXPERIMENTAL_PARALLEL_CROP_PHASE) && !defined(LW_EXPERIMENTAL_STREAMING_CROPS) +#define LW_OCR_CROP_ROWS_PER_BAND 16u + +typedef struct crop_band_context { + lw_ocr* ocr; + const uint8_t* source; + uint64_t source_byte_count; + uint32_t source_width; + uint32_t source_height; + uint32_t source_stride; +} crop_band_context; + +/* Pool callback for the parallel crop phase; workers stride over the band + * list and write disjoint crop ranges. */ +static void execute_crop_bands(void* context_void, uint32_t worker_index, + uint32_t worker_count) { + crop_band_context* context = (crop_band_context*)context_void; + lw_ocr* ocr = context->ocr; + uint32_t band_index; + for (band_index = worker_index; band_index < ocr->crop_band_count; + band_index += worker_count) { + const lw_ocr_crop_band* band = &ocr->crop_bands[band_index]; + const lw_ocr_crop* crop = &ocr->crops[band->crop_index]; + uint32_t band_width; + uint32_t band_height; + uint64_t band_bytes; + lw_status status = lw_crop_quad_bgr_u8_band( + context->source, context->source_byte_count, context->source_width, + context->source_height, context->source_stride, + &ocr->detected_boxes[crop->box_index], ocr->crop + (size_t)crop->offset, + crop->byte_count, &band_width, &band_height, &band_bytes, band->row_begin, + band->row_end); + if (status != LW_STATUS_OK && ocr->crop_band_status == LW_STATUS_OK) { + ocr->crop_band_status = status; + } + } +} + +static lw_status ensure_crop_bands(lw_ocr* ocr, uint32_t required, lw_error* error) { + if (ocr->crop_band_capacity >= required) return LW_STATUS_OK; + { + uint32_t capacity = ocr->crop_band_capacity == 0u ? 64u : ocr->crop_band_capacity; + lw_ocr_crop_band* bands; + while (capacity < required) { + if (capacity > UINT32_MAX / 2u) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "OCR crop band capacity overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + capacity *= 2u; + } + bands = (lw_ocr_crop_band*)realloc(ocr->crop_bands, + (size_t)capacity * sizeof(*bands)); + if (bands == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "OCR crop band allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + ocr->crop_bands = bands; + ocr->crop_band_capacity = capacity; + } + return LW_STATUS_OK; +} +#endif + +static lw_status run_worker_tasks(lw_ocr* ocr, const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, uint32_t crop_count, + lw_ocr_execution_profile* profile, lw_error* error) { + uint8_t worker_used[LW_OCR_MAX_WORKER_COUNT]; + uint32_t active_worker_count; + uint32_t slot; + uint64_t started = lw_ocr_profile_now(profile); + uint64_t critical_nanoseconds = 0u; + lw_status status = LW_STATUS_OK; +#if !defined(LW_EXPERIMENTAL_STREAMING_CROPS) + (void)source; + (void)source_byte_count; + (void)source_width; + (void)source_height; + (void)source_stride; +#endif + if (crop_count == 0u) + return LW_STATUS_OK; + if (crop_count > ocr->info.max_line_capacity) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid OCR worker crop schedule"); + return LW_STATUS_INVALID_ARGUMENT; + } + active_worker_count = crop_count < ocr->worker_count ? crop_count : ocr->worker_count; +#if defined(LW_EXPERIMENTAL_ADAPTIVE_REC_BUDGET) + { + /* A sparse request has fewer concurrent recognizers than the handle's + * configured line workers. Spend the same total thread budget on the + * recognizers that will actually run, before dispatching any work. */ + uint32_t rec_intra = ocr->det_intra_op_thread_cap / active_worker_count; + if (rec_intra == 0u) rec_intra = 1u; + for (slot = 0u; slot < active_worker_count; ++slot) { + lw_recognizer_set_intra_op_thread_count(ocr->recognizers[slot], rec_intra); + } + } +#endif + memset(ocr->worker_started, 0, ocr->worker_count * sizeof(*ocr->worker_started)); + memset(worker_used, 0, sizeof(worker_used)); + ocr->crop_schedule_count = crop_count; +#if defined(_WIN32) + ocr->next_crop_slot = (LONG)active_worker_count; +#else + ocr->next_crop_slot = active_worker_count; +#endif + for (slot = 0u; slot < active_worker_count; ++slot) { + uint32_t worker_index; + uint32_t candidate; + uint32_t crop_index = ocr->crop_schedule[slot]; + uint32_t target_width = ocr->crops[crop_index].target_width; + worker_index = active_worker_count; + for (candidate = 0u; candidate < active_worker_count; ++candidate) { + if (worker_used[candidate] == 0u && + lw_recognizer_current_target_width(ocr->recognizers[candidate]) == target_width) { + worker_index = candidate; + break; + } + } + if (worker_index == active_worker_count) { + for (candidate = 0u; candidate < active_worker_count; ++candidate) { + if (worker_used[candidate] == 0u) { + worker_index = candidate; + break; + } + } + } + if (worker_index == active_worker_count) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "unable to assign OCR worker"); + return LW_STATUS_INVALID_SHAPE; + } + worker_used[worker_index] = 1u; + { + lw_ocr_worker_task* task = &ocr->worker_tasks[worker_index]; + lw_crop_buffer crop_buffer = task->crop_buffer; + memset(task, 0, sizeof(*task)); + task->crop_buffer = crop_buffer; +#if defined(LW_EXPERIMENTAL_STREAMING_CROPS) + task->source = source; + task->source_byte_count = source_byte_count; + task->source_width = source_width; + task->source_height = source_height; + task->source_stride = source_stride; +#endif + task->ocr = ocr; + task->worker_index = worker_index; + task->initial_crop_slot = slot; + task->profile_enabled = 0u; + task->wall_nanoseconds = 0u; + if (profile != NULL) { + task->profile_enabled = 1u; + lw_pipeline_component_profile_reset(&task->classifier_profile, profile->clock, + profile->clock_context); + lw_pipeline_component_profile_reset(&task->recognizer_profile, profile->clock, + profile->clock_context); + } + } + } + + /* Every active worker claims the next longest remaining crop. Keep worker + * zero on the calling thread. The persistent pool runs the remaining + * workers and barriers before this returns; per-request thread creation + * stays as the fallback when the pool is unavailable. */ +#if defined(LW_EXPERIMENTAL_PERSISTENT_LINE_POOL) + if (ocr->line_pool == NULL && active_worker_count > 1u) { + ocr->line_pool = lw_thread_pool_create(ocr->worker_count); + } + if (ocr->line_pool != NULL) { + lw_thread_pool_run(ocr->line_pool, active_worker_count, execute_line_worker, ocr); + } else { + execute_worker_task(&ocr->worker_tasks[0]); + for (slot = 1u; slot < active_worker_count; ++slot) { + execute_worker_task(&ocr->worker_tasks[slot]); + } + } +#else + for (slot = 1u; slot < active_worker_count; ++slot) { + uint32_t worker_index = slot; +#if defined(_WIN32) + ocr->worker_threads[worker_index] = (HANDLE)_beginthreadex( + NULL, 0u, worker_entry, &ocr->worker_tasks[worker_index], 0u, NULL); + if (ocr->worker_threads[worker_index] != NULL) { + ocr->worker_started[worker_index] = 1u; + } else { + execute_worker_task(&ocr->worker_tasks[worker_index]); + } +#elif !defined(__EMSCRIPTEN__) + if (pthread_create(&ocr->worker_threads[worker_index], NULL, worker_entry, + &ocr->worker_tasks[worker_index]) == 0) { + ocr->worker_started[worker_index] = 1u; + } else { + execute_worker_task(&ocr->worker_tasks[worker_index]); + } +#else + execute_worker_task(&ocr->worker_tasks[worker_index]); +#endif + } + execute_worker_task(&ocr->worker_tasks[0]); + for (slot = 1u; slot < active_worker_count; ++slot) { + uint32_t worker_index = slot; + if (ocr->worker_started[worker_index] == 0u) + continue; +#if defined(_WIN32) + WaitForSingleObject(ocr->worker_threads[worker_index], INFINITE); + CloseHandle(ocr->worker_threads[worker_index]); + ocr->worker_threads[worker_index] = NULL; +#elif !defined(__EMSCRIPTEN__) + (void)pthread_join(ocr->worker_threads[worker_index], NULL); +#endif + } +#endif + if (profile != NULL) { + uint64_t finished = lw_ocr_profile_now(profile); + uint64_t batch_nanoseconds = finished >= started ? finished - started : 0u; + for (slot = 0u; slot < active_worker_count; ++slot) { + uint32_t worker_index = slot; + if (ocr->worker_tasks[worker_index].wall_nanoseconds > critical_nanoseconds) { + critical_nanoseconds = ocr->worker_tasks[worker_index].wall_nanoseconds; + } + } + lw_profile_add_value(&profile->line_workers_nanoseconds, batch_nanoseconds); + lw_profile_add_value(&profile->line_worker_critical_nanoseconds, critical_nanoseconds); + if (batch_nanoseconds > critical_nanoseconds) { + lw_profile_add_value(&profile->line_dispatch_overhead_nanoseconds, + batch_nanoseconds - critical_nanoseconds); + } + } + for (slot = 0u; slot < active_worker_count; ++slot) { + uint32_t worker_index = slot; + if (profile != NULL) { + uint32_t bucket; + lw_pipeline_component_profile_accumulate( + &profile->classifier, &ocr->worker_tasks[worker_index].classifier_profile); + lw_pipeline_component_profile_accumulate( + &profile->recognizer, &ocr->worker_tasks[worker_index].recognizer_profile); +#if defined(LW_EXPERIMENTAL_STREAMING_CROPS) + if (profile->crop_nanoseconds <= UINT64_MAX - + ocr->worker_tasks[worker_index].crop_nanoseconds) { + profile->crop_nanoseconds += ocr->worker_tasks[worker_index].crop_nanoseconds; + } else { + profile->crop_nanoseconds = UINT64_MAX; + } +#endif + lw_profile_add_value(&profile->rec_width_sample_count, + ocr->worker_tasks[worker_index].rec_width_sample_count); + lw_profile_add_value(&profile->rec_resized_width_sum, + ocr->worker_tasks[worker_index].rec_resized_width_sum); + lw_profile_add_value(&profile->rec_target_width_sum, + ocr->worker_tasks[worker_index].rec_target_width_sum); + for (bucket = 0u; bucket < LW_REC_WIDTH_HISTOGRAM_BUCKET_COUNT; ++bucket) { + lw_profile_add_value(&profile->rec_width_histogram[bucket], + ocr->worker_tasks[worker_index].rec_width_histogram[bucket]); + } + } + if (ocr->worker_tasks[worker_index].status != LW_STATUS_OK) { + status = ocr->worker_tasks[worker_index].status; + lw_set_error(error, status, ocr->worker_tasks[worker_index].error.message); + break; + } + } + for (slot = 0u; slot < ocr->worker_count; ++slot) { + if (ocr->worker_tasks != NULL) { + lw_crop_buffer_trim_after_request(&ocr->worker_tasks[slot].crop_buffer); + } + } + return status; +} + +static lw_status crop_and_run_adaptive(lw_ocr* ocr, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + uint32_t line_count, lw_ocr_execution_profile* profile, + lw_error* error) { +#if !defined(LW_EXPERIMENTAL_STREAMING_CROPS) + uint64_t crop_used = 0u; + uint64_t planned_crop_bytes = 0u; +#endif + uint32_t scheduled_count = 0u; +#if !defined(LW_EXPERIMENTAL_STREAMING_CROPS) + uint32_t planned_index; + lw_status reserve_status; + + /* Geometry has already been resolved by ocr_run_bgr_u8_impl. Reserve the + * complete aggregate once so a dense page does not repeatedly grow and + * copy the request-wide crop buffer. The per-crop check below remains a + * defensive fallback because rasterized byte counts are allowed to differ + * from the geometry estimate while earlier crops are materialized. */ + for (planned_index = 0u; planned_index < line_count; ++planned_index) { + const lw_ocr_crop* planned_crop = &ocr->crops[planned_index]; + if (planned_crop->byte_count > UINT64_MAX - planned_crop_bytes) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "aggregate OCR crop size overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + planned_crop_bytes += planned_crop->byte_count; + } + reserve_status = ensure_crop_capacity(ocr, planned_crop_bytes, error); + if (reserve_status != LW_STATUS_OK) { + return reserve_status; + } +#endif + while (scheduled_count < line_count) { + uint32_t selected_index = UINT32_MAX; + uint32_t selected_width = 0u; + uint32_t index; + lw_ocr_crop* crop; +#if !defined(LW_EXPERIMENTAL_STREAMING_CROPS) && !defined(LW_EXPERIMENTAL_PARALLEL_CROP_PHASE) + lw_detection_box* box; + uint32_t crop_width; + uint32_t crop_height; + uint64_t crop_bytes; + uint64_t crop_started; +#endif +#if !defined(LW_EXPERIMENTAL_STREAMING_CROPS) + lw_status status; +#endif + for (index = 0u; index < line_count; ++index) { + const lw_ocr_crop* candidate = &ocr->crops[index]; + if (candidate->scheduled == 0u && + (candidate->target_width > selected_width || + (candidate->target_width == selected_width && index < selected_index))) { + selected_index = index; + selected_width = candidate->target_width; + } + } + if (selected_index == UINT32_MAX) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "adaptive REC width scheduling is incomplete"); + return LW_STATUS_INVALID_SHAPE; + } + crop = &ocr->crops[selected_index]; +#if defined(LW_EXPERIMENTAL_STREAMING_CROPS) + crop->offset = 0u; + crop->scheduled = 1u; + ocr->crop_schedule[scheduled_count] = selected_index; + ++scheduled_count; +#else + if (crop_used > UINT64_MAX - crop->byte_count) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "aggregate OCR crop size overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + if (crop_used > ocr->crop_capacity || crop->byte_count > ocr->crop_capacity - crop_used) { + status = ensure_crop_capacity(ocr, crop_used + crop->byte_count, error); + if (status != LW_STATUS_OK) { + return status; + } + } +#if defined(LW_EXPERIMENTAL_PARALLEL_CROP_PHASE) + /* Serial geometry assignment plus band-list construction; the actual + * rasterization runs in the parallel phase below. */ + crop->offset = crop_used; + crop->scheduled = 1u; + ocr->crop_schedule[scheduled_count] = selected_index; + crop_used += crop->byte_count; + ++scheduled_count; + { + uint32_t unrotated_height = + lw_crop_quad_unrotated_height(&ocr->detected_boxes[crop->box_index]); + uint32_t band_row; + for (band_row = 0u; band_row < unrotated_height; + band_row += LW_OCR_CROP_ROWS_PER_BAND) { + uint32_t band_end = band_row + LW_OCR_CROP_ROWS_PER_BAND; + if (band_end > unrotated_height) band_end = unrotated_height; + status = ensure_crop_bands(ocr, ocr->crop_band_count + 1u, error); + if (status != LW_STATUS_OK) { + return status; + } + ocr->crop_bands[ocr->crop_band_count].crop_index = selected_index; + ocr->crop_bands[ocr->crop_band_count].row_begin = band_row; + ocr->crop_bands[ocr->crop_band_count].row_end = band_end; + ++ocr->crop_band_count; + } + } +#else + box = &ocr->detected_boxes[crop->box_index]; + crop_width = crop->width; + crop_height = crop->height; + crop_bytes = crop->byte_count; + crop_started = lw_ocr_profile_now(profile); + status = lw_crop_quad_bgr_u8(source, source_byte_count, source_width, source_height, + source_stride, box, ocr->crop + (size_t)crop_used, crop_bytes, + &crop_width, &crop_height, &crop_bytes); + lw_ocr_profile_add_elapsed(profile == NULL ? NULL : &profile->crop_nanoseconds, + crop_started, profile); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "OCR perspective crop failed"); + return status; + } + crop->offset = crop_used; + crop->byte_count = crop_bytes; + crop->width = crop_width; + crop->height = crop_height; + crop->scheduled = 1u; + ocr->crop_schedule[scheduled_count] = selected_index; + crop_used += crop_bytes; + ++scheduled_count; +#endif +#endif + } +#if defined(LW_EXPERIMENTAL_PARALLEL_CROP_PHASE) && !defined(LW_EXPERIMENTAL_STREAMING_CROPS) + if (ocr->crop_band_count != 0u) { + crop_band_context context; + uint32_t band_workers; + uint64_t crop_started = lw_ocr_profile_now(profile); + ocr->crop_band_status = LW_STATUS_OK; + context.ocr = ocr; + context.source = source; + context.source_byte_count = source_byte_count; + context.source_width = source_width; + context.source_height = source_height; + context.source_stride = source_stride; + if (ocr->line_pool == NULL && ocr->crop_band_count > 1u) { + ocr->line_pool = lw_thread_pool_create(ocr->worker_count); + } + band_workers = ocr->worker_count < ocr->crop_band_count ? ocr->worker_count + : ocr->crop_band_count; + if (ocr->line_pool != NULL && band_workers > 1u) { + lw_thread_pool_run(ocr->line_pool, band_workers, execute_crop_bands, &context); + } else { + execute_crop_bands(&context, 0u, 1u); + } + lw_ocr_profile_add_elapsed(profile == NULL ? NULL : &profile->crop_nanoseconds, + crop_started, profile); + { + lw_status band_status = ocr->crop_band_status; + ocr->crop_band_count = 0u; + if (band_status != LW_STATUS_OK) { + lw_set_error(error, band_status, "OCR perspective crop failed"); + return band_status; + } + } + } +#endif + return run_worker_tasks(ocr, source, source_byte_count, source_width, source_height, + source_stride, line_count, profile, error); +} + +static lw_status ocr_run_bgr_u8_impl(lw_ocr* ocr, const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, lw_ocr_line* lines, + uint32_t line_capacity, char* text_utf8, + uint64_t text_capacity, lw_ocr_result* result, + lw_ocr_execution_profile* profile, lw_error* error) { + lw_detection_result detection; + uint32_t line_count = 0u; + uint64_t text_used = 0u; + uint64_t total_started; + uint64_t output_started; + uint64_t crop_setup_started; + uint32_t index; + int capacity_query; + lw_ocr_result output; + uint32_t result_size; + lw_status status; + if (ocr == NULL || source == NULL || result == NULL || result->struct_size < sizeof(uint32_t) || + (lines == NULL && line_capacity != 0u) || (text_utf8 == NULL && text_capacity != 0u)) { + lw_set_error( + error, LW_STATUS_INVALID_ARGUMENT, + "OCR handle, BGR source, initialized result, and valid output buffers are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + result_size = result->struct_size; + clear_result(&output); + total_started = lw_ocr_profile_now(profile); + capacity_query = + lines == NULL && line_capacity == 0u && text_utf8 == NULL && text_capacity == 0u; + + /* DET writes into handle-owned scratch space so a capacity-only call still + * executes the exact same pipeline and reports exact output requirements. */ + lw_detection_result_init(&detection); + status = profile == NULL + ? lw_detector_detect_bgr_u8(ocr->detector, source, source_byte_count, source_width, + source_height, source_stride, ocr->detected_boxes, + ocr->info.max_line_capacity, &detection, error) + : lw_detector_detect_bgr_u8_profiled( + ocr->detector, source, source_byte_count, source_width, source_height, + source_stride, ocr->detected_boxes, ocr->info.max_line_capacity, &detection, + &profile->detector, error); + output.detected_count = detection.box_count; + output.detector_resized_width = detection.resized_width; + output.detector_resized_height = detection.resized_height; + if (status != LW_STATUS_OK) { + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return status; + } + /* Resolve crop geometry before running REC. The private scheduler can then + * materialize every crop in longest-first order and dynamically balance + * the complete request across the available line workers. */ + crop_setup_started = lw_ocr_profile_now(profile); + for (index = 0u; index < detection.box_count; ++index) { + lw_detection_box* box = &ocr->detected_boxes[index]; + lw_ocr_crop* crop; + uint32_t crop_width; + uint32_t crop_height; + uint64_t crop_bytes; + uint64_t crop_pixels; + status = lw_crop_quad_size(box, &crop_width, &crop_height, &crop_bytes); + if (status == LW_STATUS_INVALID_SHAPE) + continue; + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "unable to compute OCR crop dimensions"); + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return status; + } + crop_pixels = (uint64_t)crop_width * crop_height; + if (crop_pixels > ocr->max_crop_pixels) { + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, "OCR crop exceeds max_crop_pixels"); + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return LW_STATUS_MEMORY_LIMIT; + } + if (crop_width > UINT32_MAX / 3u) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "OCR crop row stride overflows"); + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return LW_STATUS_OUT_OF_BOUNDS; + } + crop = &ocr->crops[line_count]; + crop->offset = 0u; + crop->byte_count = crop_bytes; + crop->width = crop_width; + crop->height = crop_height; + crop->box_index = index; + crop->target_width = + lw_recognizer_target_width_for_image(ocr->recognizers[0], crop_width, crop_height); + crop->scheduled = 0u; + ++line_count; + } + lw_ocr_profile_add_elapsed(profile == NULL ? NULL : &profile->crop_setup_nanoseconds, + crop_setup_started, profile); + status = crop_and_run_adaptive(ocr, source, source_byte_count, source_width, source_height, + source_stride, line_count, profile, error); + if (status != LW_STATUS_OK) { + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return status; + } + /* Workers write fixed-size text slots to avoid synchronization. Compact + * those slots in reading order before publishing the public result. */ + output_started = lw_ocr_profile_now(profile); + for (index = 0u; index < line_count; ++index) { + lw_ocr_line* line = &ocr->scratch_lines[index]; + uint64_t line_bytes = line->text_length + 1u; + if (text_used > ocr->info.max_text_capacity - line_bytes) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "OCR text scratch capacity is exhausted"); + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return LW_STATUS_OUT_OF_BOUNDS; + } + memmove(ocr->scratch_text + (size_t)text_used, + ocr->scratch_text + (size_t)line->text_offset, (size_t)line_bytes); + line->text_offset = text_used; + text_used += line_bytes; + } + output.line_count = line_count; + output.required_line_capacity = line_count; + output.required_text_capacity = text_used; + if (capacity_query) { + /* The caller can now allocate exact line/text buffers and call again. */ + lw_ocr_profile_add_elapsed(profile == NULL ? NULL : &profile->output_nanoseconds, + output_started, profile); + lw_ocr_profile_add_elapsed(profile == NULL ? NULL : &profile->total_nanoseconds, + total_started, profile); + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + } + if (line_capacity < line_count || text_capacity < text_used || + (line_count != 0u && lines == NULL) || (text_used != 0u && text_utf8 == NULL)) { + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, + "OCR line or text output capacity is insufficient"); + return LW_STATUS_OUT_OF_BOUNDS; + } + /* Publish only after every line succeeded, preventing partially filled + * caller buffers from looking like a successful OCR response. */ + if (line_count != 0u) + memcpy(lines, ocr->scratch_lines, (size_t)line_count * sizeof(*lines)); + if (text_used != 0u) + memcpy(text_utf8, ocr->scratch_text, (size_t)text_used); + lw_ocr_profile_add_elapsed(profile == NULL ? NULL : &profile->output_nanoseconds, + output_started, profile); + lw_ocr_profile_add_elapsed(profile == NULL ? NULL : &profile->total_nanoseconds, total_started, + profile); + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +lw_status lw_ocr_run_bgr_u8(lw_ocr* ocr, const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, uint32_t source_stride, + lw_ocr_line* lines, uint32_t line_capacity, char* text_utf8, + uint64_t text_capacity, lw_ocr_result* result, lw_error* error) { + return ocr_run_bgr_u8_impl(ocr, source, source_byte_count, source_width, source_height, + source_stride, lines, line_capacity, text_utf8, text_capacity, + result, NULL, error); +} + +lw_status lw_ocr_run_bgr_u8_profiled(lw_ocr* ocr, const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, lw_ocr_line* lines, + uint32_t line_capacity, char* text_utf8, + uint64_t text_capacity, lw_ocr_result* result, + lw_ocr_execution_profile* profile, lw_error* error) { + if (profile == NULL || profile->struct_size != sizeof(*profile) || profile->reserved != 0u || + profile->clock == NULL || + profile->detector.execution.struct_size != sizeof(profile->detector.execution) || + profile->classifier.execution.struct_size != sizeof(profile->classifier.execution) || + profile->recognizer.execution.struct_size != sizeof(profile->recognizer.execution)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "an initialized full OCR profile and clock are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + return ocr_run_bgr_u8_impl(ocr, source, source_byte_count, source_width, source_height, + source_stride, lines, line_capacity, text_utf8, text_capacity, + result, profile, error); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/ocr_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/ocr_internal.h new file mode 100644 index 00000000..3a37005d --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/ocr_internal.h @@ -0,0 +1,17 @@ +#ifndef LW_OCR_INTERNAL_H +#define LW_OCR_INTERNAL_H + +/* Private controls and diagnostics for benchmarks and regression tests. They + * intentionally stay outside the installed C ABI until the policy is stable. */ + +#include "cpu_topology.h" +#include "lw_infer.h" + +#include + +void lw_ocr_set_det_intra_op_thread_count(lw_ocr* ocr, uint32_t thread_count); +uint32_t lw_ocr_get_det_intra_op_thread_count(const lw_ocr* ocr); +uint32_t lw_ocr_get_det_intra_op_thread_cap(const lw_ocr* ocr); +void lw_ocr_get_cpu_topology(const lw_ocr* ocr, lw_cpu_topology* topology); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/profile.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/profile.c new file mode 100644 index 00000000..54474c61 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/profile.c @@ -0,0 +1,244 @@ +#include "profile_internal.h" + +#include +#include +#if defined(__EMSCRIPTEN__) +#include + +/* The C environment is virtual in Emscripten and does not inherit Node's + * process.env. Browser builds have no process, so profiling stays disabled. */ +EM_JS(int, lw_node_profile_env_present, (const char* name), { + return typeof process !== 'undefined' && process.env && + process.env[UTF8ToString(name)] !== undefined ? 1 : 0; +}) +EM_JS(int, lw_node_profile_env_is_one, (const char* name), { + return typeof process !== 'undefined' && process.env && + process.env[UTF8ToString(name)] === '1' ? 1 : 0; +}) +#endif + +int lw_profile_env_present(const char* name) { +#if defined(__EMSCRIPTEN__) + return name != NULL && lw_node_profile_env_present(name); +#elif defined(_MSC_VER) + char* value = NULL; + size_t length = 0u; + int present; + if (name == NULL || _dupenv_s(&value, &length, name) != 0) return 0; + present = value != NULL; + free(value); + return present; +#else + return name != NULL && getenv(name) != NULL; +#endif +} + +int lw_profile_env_is_one(const char* name) { +#if defined(__EMSCRIPTEN__) + return name != NULL && lw_node_profile_env_is_one(name); +#elif defined(_MSC_VER) + char* value = NULL; + size_t length = 0u; + int enabled; + if (name == NULL || _dupenv_s(&value, &length, name) != 0) return 0; + enabled = value != NULL && value[0] == '1' && value[1] == '\0'; + free(value); + return enabled; +#else + const char* value = name == NULL ? NULL : getenv(name); + return value != NULL && value[0] == '1' && value[1] == '\0'; +#endif +} + +static void add_saturated(uint64_t* destination, uint64_t value) { + if (*destination > UINT64_MAX - value) { + *destination = UINT64_MAX; + } else { + *destination += value; + } +} + +void lw_profile_add_value(uint64_t* destination, uint64_t value) { + if (destination != NULL) { + add_saturated(destination, value); + } +} + +uint32_t lw_rec_width_histogram_bucket(uint32_t resized_width) { + static const uint32_t upper_bounds[LW_REC_WIDTH_HISTOGRAM_BUCKET_COUNT - 1u] = { + 192u, 256u, 320u, 480u, 640u, 800u, 960u}; + uint32_t bucket = 0u; + while (bucket + 1u < LW_REC_WIDTH_HISTOGRAM_BUCKET_COUNT && + resized_width > upper_bounds[bucket]) { + ++bucket; + } + return bucket; +} + +void lw_pipeline_component_profile_reset(lw_pipeline_component_profile* profile, + lw_execution_profile_clock clock, void* clock_context) { + if (profile == NULL) { + return; + } + memset(profile, 0, sizeof(*profile)); + profile->execution.struct_size = (uint32_t)sizeof(profile->execution); + profile->execution.clock = clock; + profile->execution.clock_context = clock_context; +} + +void lw_ocr_execution_profile_init(lw_ocr_execution_profile* profile, + lw_execution_profile_clock clock, void* clock_context) { + if (profile == NULL) { + return; + } + memset(profile, 0, sizeof(*profile)); + profile->struct_size = (uint32_t)sizeof(*profile); + profile->clock = clock; + profile->clock_context = clock_context; + lw_pipeline_component_profile_reset(&profile->detector, clock, clock_context); + lw_pipeline_component_profile_reset(&profile->classifier, clock, clock_context); + lw_pipeline_component_profile_reset(&profile->recognizer, clock, clock_context); +} + +void lw_pipeline_component_profile_accumulate(lw_pipeline_component_profile* destination, + const lw_pipeline_component_profile* source) { + uint32_t index; + if (destination == NULL || source == NULL) { + return; + } + add_saturated(&destination->preprocess_nanoseconds, source->preprocess_nanoseconds); + add_saturated(&destination->graph_nanoseconds, source->graph_nanoseconds); + add_saturated(&destination->postprocess_nanoseconds, source->postprocess_nanoseconds); + add_saturated(&destination->unclip_nanoseconds, source->unclip_nanoseconds); + add_saturated(&destination->session_cache_hits, source->session_cache_hits); + add_saturated(&destination->session_cache_misses, source->session_cache_misses); + add_saturated(&destination->session_reconfigurations, source->session_reconfigurations); + add_saturated(&destination->compiled_backend_lines, source->compiled_backend_lines); + add_saturated(&destination->canonical_fallback_lines, source->canonical_fallback_lines); + add_saturated(&destination->compiled_backend_runs, source->compiled_backend_runs); + add_saturated(&destination->canonical_fallback_runs, source->canonical_fallback_runs); + for (index = 0u; index < LW_REC_WIDTH_HISTOGRAM_BUCKET_COUNT; ++index) { + uint32_t node; + for (node = 0u; node < LW_EXECUTION_PROFILE_NODE_CAPACITY; ++node) { + add_saturated(&destination->node_nanoseconds_by_width[index][node], + source->node_nanoseconds_by_width[index][node]); + add_saturated(&destination->node_invocations_by_width[index][node], + source->node_invocations_by_width[index][node]); + } + } + for (index = 0u; index < LW_EXECUTION_PROFILE_OPERATOR_CAPACITY; ++index) { + add_saturated(&destination->execution.operator_nanoseconds[index], + source->execution.operator_nanoseconds[index]); + add_saturated(&destination->execution.operator_invocations[index], + source->execution.operator_invocations[index]); + } + for (index = 0u; index < LW_EXECUTION_PROFILE_NODE_CAPACITY; ++index) { + add_saturated(&destination->execution.node_nanoseconds[index], + source->execution.node_nanoseconds[index]); + add_saturated(&destination->execution.node_invocations[index], + source->execution.node_invocations[index]); + } + for (index = 0u; index < LW_EXECUTION_PROFILE_CONV_CLASS_CAPACITY; ++index) { + add_saturated(&destination->execution.conv_class_nanoseconds[index], + source->execution.conv_class_nanoseconds[index]); + add_saturated(&destination->execution.conv_class_invocations[index], + source->execution.conv_class_invocations[index]); + } + for (index = 0u; index < LW_EXECUTION_PROFILE_THREAD_HISTOGRAM_CAPACITY; ++index) { + add_saturated(&destination->execution.conv_thread_histogram[index], + source->execution.conv_thread_histogram[index]); + add_saturated(&destination->execution.conv_transpose_thread_histogram[index], + source->execution.conv_transpose_thread_histogram[index]); + } + add_saturated(&destination->execution.prepared_binding_lookups, + source->execution.prepared_binding_lookups); + add_saturated(&destination->execution.prepared_binding_hits, + source->execution.prepared_binding_hits); + add_saturated(&destination->execution.prepared_binding_fallbacks, + source->execution.prepared_binding_fallbacks); + add_saturated(&destination->execution.prepared_node_invocations, + source->execution.prepared_node_invocations); + add_saturated(&destination->execution.generic_node_invocations, + source->execution.generic_node_invocations); + add_saturated(&destination->execution.prepared_conv1x1_invocations, + source->execution.prepared_conv1x1_invocations); + add_saturated(&destination->execution.prepared_conv3x3_invocations, + source->execution.prepared_conv3x3_invocations); + add_saturated(&destination->execution.packed_conv1x1_invocations, + source->execution.packed_conv1x1_invocations); + add_saturated(&destination->execution.packed_conv3x3_stride2_invocations, + source->execution.packed_conv3x3_stride2_invocations); + add_saturated(&destination->execution.unpacked_conv_invocations, + source->execution.unpacked_conv_invocations); + add_saturated(&destination->execution.packed_matmul_invocations, + source->execution.packed_matmul_invocations); + add_saturated(&destination->execution.unpacked_matmul_invocations, + source->execution.unpacked_matmul_invocations); + add_saturated(&destination->execution.fused_gelu_invocations, + source->execution.fused_gelu_invocations); + add_saturated(&destination->execution.ctc_greedy_invocations, + source->execution.ctc_greedy_invocations); + add_saturated(&destination->execution.ctc_packed_projection_invocations, + source->execution.ctc_packed_projection_invocations); + add_saturated(&destination->execution.ctc_generic_projection_invocations, + source->execution.ctc_generic_projection_invocations); + add_saturated(&destination->execution.layout_transform_nanoseconds, source->execution.layout_transform_nanoseconds); + add_saturated(&destination->execution.layout_transform_invocations, source->execution.layout_transform_invocations); + add_saturated(&destination->execution.layout_transform_bytes, source->execution.layout_transform_bytes); + if (destination->execution.layout_analysis_runs == 0u && source->execution.layout_analysis_runs != 0u) { + destination->execution.layout_candidate_nodes = source->execution.layout_candidate_nodes; + destination->execution.layout_selected_nodes = source->execution.layout_selected_nodes; + destination->execution.layout_fallback_nodes = source->execution.layout_fallback_nodes; + destination->execution.layout_analysis_runs = 1u; + } +} + +void lw_pipeline_profile_capture_node_width_delta( + lw_pipeline_component_profile* profile, uint32_t width_bucket, + const uint64_t before_nanoseconds[LW_EXECUTION_PROFILE_NODE_CAPACITY], + const uint64_t before_invocations[LW_EXECUTION_PROFILE_NODE_CAPACITY]) { + uint32_t node; + if (profile == NULL || before_nanoseconds == NULL || before_invocations == NULL || + width_bucket >= LW_REC_WIDTH_HISTOGRAM_BUCKET_COUNT) { + return; + } + for (node = 0u; node < LW_EXECUTION_PROFILE_NODE_CAPACITY; ++node) { + uint64_t current_nanoseconds = profile->execution.node_nanoseconds[node]; + uint64_t current_invocations = profile->execution.node_invocations[node]; + uint64_t elapsed = current_nanoseconds >= before_nanoseconds[node] + ? current_nanoseconds - before_nanoseconds[node] + : UINT64_MAX - before_nanoseconds[node] + current_nanoseconds + 1u; + uint64_t invocations = current_invocations >= before_invocations[node] + ? current_invocations - before_invocations[node] + : UINT64_MAX - before_invocations[node] + + current_invocations + 1u; + add_saturated(&profile->node_nanoseconds_by_width[width_bucket][node], elapsed); + add_saturated(&profile->node_invocations_by_width[width_bucket][node], invocations); + } +} + +uint64_t lw_pipeline_profile_now(const lw_pipeline_component_profile* profile) { + return profile == NULL || profile->execution.clock == NULL + ? 0u + : profile->execution.clock(profile->execution.clock_context); +} + +void lw_pipeline_profile_add_elapsed(uint64_t* destination, uint64_t started, + const lw_pipeline_component_profile* profile) { + uint64_t finished = lw_pipeline_profile_now(profile); + if (destination != NULL && finished >= started) { + add_saturated(destination, finished - started); + } +} + +uint64_t lw_ocr_profile_now(const lw_ocr_execution_profile* profile) { + return profile == NULL || profile->clock == NULL ? 0u : profile->clock(profile->clock_context); +} + +void lw_ocr_profile_add_elapsed(uint64_t* destination, uint64_t started, + const lw_ocr_execution_profile* profile) { + uint64_t finished = lw_ocr_profile_now(profile); + if (destination != NULL && finished >= started) { + add_saturated(destination, finished - started); + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/profile_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/profile_internal.h new file mode 100644 index 00000000..afc7e1e0 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/profile_internal.h @@ -0,0 +1,104 @@ +#ifndef LW_PPOCR_PROFILE_INTERNAL_H +#define LW_PPOCR_PROFILE_INTERNAL_H + +#include "executor_internal.h" +#include "lw_infer.h" + +#include + +#define LW_REC_WIDTH_HISTOGRAM_BUCKET_COUNT 8u + +/* + * Private profiling contract used by benchmarks and regression tests. It is + * deliberately kept out of the public C ABI until the measurement fields and + * their wall-time/work-time semantics are proven stable. + */ +typedef struct lw_pipeline_component_profile { + uint64_t preprocess_nanoseconds; + uint64_t graph_nanoseconds; + uint64_t postprocess_nanoseconds; + uint64_t unclip_nanoseconds; + uint64_t session_cache_hits; + uint64_t session_cache_misses; + uint64_t session_reconfigurations; + uint64_t compiled_backend_lines; + uint64_t canonical_fallback_lines; + uint64_t compiled_backend_runs; + uint64_t canonical_fallback_runs; + uint64_t node_nanoseconds_by_width[LW_REC_WIDTH_HISTOGRAM_BUCKET_COUNT] + [LW_EXECUTION_PROFILE_NODE_CAPACITY]; + uint64_t node_invocations_by_width[LW_REC_WIDTH_HISTOGRAM_BUCKET_COUNT] + [LW_EXECUTION_PROFILE_NODE_CAPACITY]; + lw_execution_profile execution; +} lw_pipeline_component_profile; + +typedef struct lw_ocr_execution_profile { + uint32_t struct_size; + uint32_t reserved; + lw_execution_profile_clock clock; + void* clock_context; + uint64_t total_nanoseconds; + uint64_t crop_nanoseconds; + uint64_t line_workers_nanoseconds; + uint64_t line_worker_critical_nanoseconds; + uint64_t line_dispatch_overhead_nanoseconds; + uint64_t crop_setup_nanoseconds; + uint64_t output_nanoseconds; + uint64_t rec_width_sample_count; + uint64_t rec_resized_width_sum; + uint64_t rec_target_width_sum; + uint64_t rec_width_histogram[LW_REC_WIDTH_HISTOGRAM_BUCKET_COUNT]; + lw_pipeline_component_profile detector; + lw_pipeline_component_profile classifier; + lw_pipeline_component_profile recognizer; +} lw_ocr_execution_profile; + +void lw_ocr_execution_profile_init(lw_ocr_execution_profile* profile, + lw_execution_profile_clock clock, void* clock_context); +void lw_pipeline_component_profile_reset(lw_pipeline_component_profile* profile, + lw_execution_profile_clock clock, void* clock_context); +void lw_pipeline_component_profile_accumulate(lw_pipeline_component_profile* destination, + const lw_pipeline_component_profile* source); +uint64_t lw_pipeline_profile_now(const lw_pipeline_component_profile* profile); +void lw_pipeline_profile_add_elapsed(uint64_t* destination, uint64_t started, + const lw_pipeline_component_profile* profile); +uint64_t lw_ocr_profile_now(const lw_ocr_execution_profile* profile); +void lw_ocr_profile_add_elapsed(uint64_t* destination, uint64_t started, + const lw_ocr_execution_profile* profile); +void lw_profile_add_value(uint64_t* destination, uint64_t value); +int lw_profile_env_present(const char* name); +int lw_profile_env_is_one(const char* name); +uint32_t lw_rec_width_histogram_bucket(uint32_t resized_width); +void lw_pipeline_profile_capture_node_width_delta( + lw_pipeline_component_profile* profile, uint32_t width_bucket, + const uint64_t before_nanoseconds[LW_EXECUTION_PROFILE_NODE_CAPACITY], + const uint64_t before_invocations[LW_EXECUTION_PROFILE_NODE_CAPACITY]); + +lw_status lw_detector_detect_bgr_u8_profiled( + lw_detector* detector, const uint8_t* source, uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, lw_detection_box* boxes, uint32_t box_capacity, + lw_detection_result* result, lw_pipeline_component_profile* profile, lw_error* error); + +lw_status lw_classifier_classify_bgr_u8_profiled(lw_classifier* classifier, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + lw_classification_result* result, + lw_pipeline_component_profile* profile, + lw_error* error); + +lw_status lw_recognizer_recognize_bgr_u8_profiled(lw_recognizer* recognizer, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + char* text_utf8, uint64_t text_capacity, + lw_recognition_result* result, + lw_pipeline_component_profile* profile, + lw_error* error); + +lw_status lw_ocr_run_bgr_u8_profiled(lw_ocr* ocr, const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, lw_ocr_line* lines, + uint32_t line_capacity, char* text_utf8, + uint64_t text_capacity, lw_ocr_result* result, + lw_ocr_execution_profile* profile, lw_error* error); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/reading_order.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/reading_order.c new file mode 100644 index 00000000..129557f0 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/reading_order.c @@ -0,0 +1,194 @@ +#include "reading_order.h" + +#include +#include +#include +#include + +typedef struct reading_item { + lw_detection_box box; + uint32_t original_index; + uint32_t column; + float min_x; + float max_x; + float center_x; + float min_y; + float center_y; + float width; +} reading_item; + +typedef struct reading_column { + float min_x; + float max_x; + float center_x; + float width; + uint32_t count; +} reading_column; + +/* Keep the count comparison wider than uint32_t. GCC otherwise diagnoses a + * valid bounded allocation check as always false on 64-bit targets. */ +static int allocation_fits(uint64_t count, size_t element_size) { + return element_size != 0u && count <= (uint64_t)(SIZE_MAX / element_size); +} + +int lw_reading_order_is_valid(uint32_t reading_order) { + return reading_order <= (uint32_t)LW_READING_ORDER_VERTICAL_LTR; +} + +static void legacy_horizontal_sort(lw_detection_box* boxes, uint32_t count) { + uint32_t index; + for (index = 1u; index < count; ++index) { + lw_detection_box value = boxes[index]; + uint32_t position = index; + while (position > 0u) { + lw_detection_box* previous = &boxes[position - 1u]; + float delta_y = previous->y1 - value.y1; + int comes_after = fabsf(delta_y) < 10.0f ? previous->x1 > value.x1 + : previous->y1 > value.y1; + if (!comes_after) + break; + boxes[position] = *previous; + --position; + } + boxes[position] = value; + } +} + +static void item_geometry(reading_item* item) { + const lw_detection_box* box = &item->box; + item->min_x = item->max_x = box->x1; + item->min_y = box->y1; + if (box->x2 < item->min_x) item->min_x = box->x2; + if (box->x3 < item->min_x) item->min_x = box->x3; + if (box->x4 < item->min_x) item->min_x = box->x4; + if (box->x2 > item->max_x) item->max_x = box->x2; + if (box->x3 > item->max_x) item->max_x = box->x3; + if (box->x4 > item->max_x) item->max_x = box->x4; + item->min_y = box->y1; + if (box->y2 < item->min_y) item->min_y = box->y2; + if (box->y3 < item->min_y) item->min_y = box->y3; + if (box->y4 < item->min_y) item->min_y = box->y4; + item->center_x = (item->min_x + item->max_x) * 0.5f; + item->center_y = (box->y1 + box->y2 + box->y3 + box->y4) * 0.25f; + item->width = item->max_x - item->min_x; + if (!(item->width > 0.0f) || !isfinite(item->width)) item->width = 1.0f; +} + +static int item_x_after(const reading_item* left, const reading_item* right) { + if (left->center_x > right->center_x) return 1; + if (left->center_x < right->center_x) return 0; + return left->original_index > right->original_index; +} + +static float interval_overlap(const reading_item* item, const reading_column* column) { + float overlap = fminf(item->max_x, column->max_x) - fmaxf(item->min_x, column->min_x); + float denominator = fmaxf(item->width, column->width); + return overlap > 0.0f && denominator > 0.0f ? overlap / denominator : 0.0f; +} + +static int item_before(const reading_item* left, const reading_item* right, + const reading_column* columns, uint32_t reverse_columns) { + const reading_column* left_column = &columns[left->column]; + const reading_column* right_column = &columns[right->column]; + if (left->column != right->column) { + if (left_column->center_x != right_column->center_x) + return reverse_columns ? left_column->center_x > right_column->center_x + : left_column->center_x < right_column->center_x; + return left->column < right->column; + } + if (left->min_y != right->min_y) return left->min_y < right->min_y; + if (left->center_y != right->center_y) return left->center_y < right->center_y; + return left->original_index < right->original_index; +} + +static lw_status vertical_sort(lw_detection_box* boxes, uint32_t count, + uint32_t reading_order) { + reading_item* items; + reading_column* columns; + uint32_t column_count = 0u; + uint32_t index; + if (count < 2u) return LW_STATUS_OK; + if (!allocation_fits(count, sizeof(*items)) || + !allocation_fits(count, sizeof(*columns))) + return LW_STATUS_OUT_OF_BOUNDS; + items = (reading_item*)malloc((size_t)count * sizeof(*items)); + columns = (reading_column*)calloc(count, sizeof(*columns)); + if (items == NULL || columns == NULL) { + free(columns); + free(items); + return LW_STATUS_OUT_OF_MEMORY; + } + for (index = 0u; index < count; ++index) { + items[index].box = boxes[index]; + items[index].original_index = index; + item_geometry(&items[index]); + } + for (index = 1u; index < count; ++index) { + reading_item value = items[index]; + uint32_t position = index; + while (position > 0u && item_x_after(&items[position - 1u], &value)) { + items[position] = items[position - 1u]; + --position; + } + items[position] = value; + } + for (index = 0u; index < count; ++index) { + uint32_t column; + uint32_t best = UINT32_MAX; + float best_overlap = 0.0f; + float best_distance = INFINITY; + for (column = 0u; column < column_count; ++column) { + float overlap = interval_overlap(&items[index], &columns[column]); + float distance = fabsf(items[index].center_x - columns[column].center_x); + if (overlap >= 0.45f && + (overlap > best_overlap || + (overlap == best_overlap && distance < best_distance))) { + best = column; + best_overlap = overlap; + best_distance = distance; + } + } + if (best == UINT32_MAX) { + best = column_count++; + columns[best].min_x = items[index].min_x; + columns[best].max_x = items[index].max_x; + columns[best].center_x = items[index].center_x; + columns[best].width = items[index].width; + } + items[index].column = best; + columns[best].min_x = fminf(columns[best].min_x, items[index].min_x); + columns[best].max_x = fmaxf(columns[best].max_x, items[index].max_x); + columns[best].count += 1u; + columns[best].center_x = + (columns[best].center_x * (float)(columns[best].count - 1u) + + items[index].center_x) / (float)columns[best].count; + columns[best].width = fmaxf(columns[best].width, items[index].width); + } + for (index = 1u; index < count; ++index) { + reading_item value = items[index]; + uint32_t position = index; + while (position > 0u && + item_before(&value, &items[position - 1u], columns, + reading_order == LW_READING_ORDER_VERTICAL_RTL)) { + items[position] = items[position - 1u]; + --position; + } + items[position] = value; + } + for (index = 0u; index < count; ++index) boxes[index] = items[index].box; + free(columns); + free(items); + return LW_STATUS_OK; +} + +lw_status lw_sort_detection_boxes(lw_detection_box* boxes, uint32_t count, + uint32_t reading_order) { + if ((boxes == NULL && count != 0u) || !lw_reading_order_is_valid(reading_order)) + return LW_STATUS_INVALID_ARGUMENT; + if (count < 2u) return LW_STATUS_OK; + if (reading_order == LW_READING_ORDER_HORIZONTAL_LTR) { + legacy_horizontal_sort(boxes, count); + return LW_STATUS_OK; + } + return vertical_sort(boxes, count, reading_order); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/reading_order.h b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/reading_order.h new file mode 100644 index 00000000..9712d3d6 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/reading_order.h @@ -0,0 +1,10 @@ +#ifndef LW_READING_ORDER_INTERNAL_H +#define LW_READING_ORDER_INTERNAL_H + +#include "lw_infer.h" +#include + +int lw_reading_order_is_valid(uint32_t reading_order); +lw_status lw_sort_detection_boxes(lw_detection_box* boxes, uint32_t count, uint32_t reading_order); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/rec_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/rec_internal.h new file mode 100644 index 00000000..66c226dc --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/rec_internal.h @@ -0,0 +1,82 @@ +#ifndef LW_REC_INTERNAL_H +#define LW_REC_INTERNAL_H + +/* Private REC preprocessing, dictionary and CTC-decoding contracts. */ + +#include "lw_infer.h" + +#include + +#define LW_REC_INPUT_HEIGHT 48u + +typedef struct lw_rec_dictionary lw_rec_dictionary; +typedef struct lw_model lw_model; + +/* Full OCR may treat the public REC target width as a maximum and select a + * smaller graph width for short crops. These helpers stay private so the + * stable C ABI and recognition-only behavior remain unchanged. */ +lw_status lw_recognizer_enable_adaptive_width(lw_recognizer* recognizer, uint32_t enabled, + lw_error* error); +lw_status lw_recognizer_enable_resident_widths(lw_recognizer* recognizer, lw_error* error); +uint32_t lw_recognizer_target_width_for_image(const lw_recognizer* recognizer, + uint32_t source_width, uint32_t source_height); +uint32_t lw_recognizer_current_target_width(const lw_recognizer* recognizer); + +lw_status lw_rec_preprocess_bgr_u8_nhwc(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, uint32_t target_width, float* output, + uint64_t output_element_count, uint32_t* resized_width); + +lw_status lw_rec_preprocess_bgr_u8(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, uint32_t target_width, float* output, + uint64_t output_element_count, uint32_t* resized_width); + +lw_status lw_rec_dictionary_load(const char* path_utf8, lw_rec_dictionary** out_dictionary, + lw_error* error); +void lw_rec_dictionary_free(lw_rec_dictionary* dictionary); +void lw_rec_dictionary_retain(lw_rec_dictionary* dictionary); +uint32_t lw_rec_dictionary_class_count(const lw_rec_dictionary* dictionary); +uint32_t lw_rec_dictionary_max_label_byte_count(const lw_rec_dictionary* dictionary); + +lw_status lw_rec_ctc_decode_f32(const lw_rec_dictionary* dictionary, const float* probabilities, + uint64_t probability_element_count, uint32_t time_steps, + uint32_t class_count, char* text_utf8, uint64_t text_capacity, + uint64_t* required_capacity, float* score, uint32_t* emitted_count, + lw_error* error); +lw_status lw_rec_ctc_decode_known_capacity_f32(const lw_rec_dictionary* dictionary, + const float* probabilities, + uint64_t probability_element_count, + uint32_t time_steps, uint32_t class_count, + char* text_utf8, uint64_t text_capacity, + uint64_t* required_capacity, float* score, + uint32_t* emitted_count, lw_error* error); +lw_status lw_rec_ctc_decode_greedy_f32(const lw_rec_dictionary* dictionary, + const uint32_t* best_indices, + const float* best_probabilities, uint32_t time_steps, + uint32_t class_count, char* text_utf8, + uint64_t text_capacity, uint64_t* required_capacity, + float* score, uint32_t* emitted_count, lw_error* error); +lw_status lw_rec_ctc_decode_greedy_known_capacity_f32( + const lw_rec_dictionary* dictionary, const uint32_t* best_indices, + const float* best_probabilities, uint32_t time_steps, uint32_t class_count, + char* text_utf8, uint64_t text_capacity, uint64_t* required_capacity, float* score, + uint32_t* emitted_count, lw_error* error); + +lw_status lw_recognizer_clone(const lw_recognizer* source, lw_recognizer** out_recognizer, + lw_error* error); + +/* Enable intra-op parallelism for the recognizer's CTC head projection. + * thread_count <= 1 keeps the serial path. */ +void lw_recognizer_set_intra_op_thread_count(lw_recognizer* recognizer, uint32_t thread_count); + +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +/* Test-only internal hook: disable the compiled x64 REC backend so the same + * binary can measure canonical recognizer timing. */ +void lw_recognizer_test_disable_x64_backend(lw_recognizer* recognizer); +/* Private lazy-fallback contract probe; not part of the public C ABI. */ +int lw_recognizer_test_has_canonical_session(const lw_recognizer* recognizer); +uint32_t lw_recognizer_test_compiled_width_count(const lw_recognizer* recognizer); +#endif + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/rec_preprocess.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/rec_preprocess.c new file mode 100644 index 00000000..c36f512c --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/rec_preprocess.c @@ -0,0 +1,201 @@ +#include "rec_internal.h" + +/* Resize/pad a text-line crop and normalize it for the REC model. */ + +#include +#include +#include + +static uint32_t clamp_coordinate(int64_t coordinate, uint32_t limit) { + if (coordinate < 0) { + return 0u; + } + if ((uint64_t)coordinate >= limit) { + return limit - 1u; + } + return (uint32_t)coordinate; +} + +lw_status lw_rec_preprocess_bgr_u8(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, uint32_t target_width, float* output, + uint64_t output_element_count, uint32_t* resized_width) { + uint64_t row_bytes; + uint64_t required_source_bytes; + uint64_t required_output_elements; + uint64_t scaled_width_numerator; + uint64_t computed_width; + uint32_t actual_width; + uint64_t channel_plane; + uint32_t channel; + uint32_t output_y; + const double normalize_scale = 2.0 / 255.0; + + if (resized_width != NULL) { + *resized_width = 0u; + } + if (source == NULL || output == NULL || source_width == 0u || source_height == 0u || + target_width == 0u) { + return LW_STATUS_INVALID_ARGUMENT; + } + row_bytes = (uint64_t)source_width * 3u; + if (row_bytes > UINT32_MAX || source_stride < row_bytes || + (uint64_t)(source_height - 1u) * source_stride > UINT64_MAX - row_bytes) { + return LW_STATUS_INVALID_SHAPE; + } + required_source_bytes = (uint64_t)(source_height - 1u) * source_stride + row_bytes; + required_output_elements = (uint64_t)3u * LW_REC_INPUT_HEIGHT * target_width; + if (required_source_bytes > SIZE_MAX || source_byte_count < required_source_bytes || + output_element_count != required_output_elements || + required_output_elements > SIZE_MAX / sizeof(float)) { + return LW_STATUS_INVALID_SHAPE; + } + if ((uint64_t)source_width > (UINT64_MAX - (source_height - 1u)) / LW_REC_INPUT_HEIGHT) { + return LW_STATUS_OUT_OF_BOUNDS; + } + scaled_width_numerator = (uint64_t)LW_REC_INPUT_HEIGHT * source_width; + computed_width = (scaled_width_numerator + source_height - 1u) / source_height; + actual_width = computed_width > target_width ? target_width : (uint32_t)computed_width; + channel_plane = (uint64_t)LW_REC_INPUT_HEIGHT * target_width; + + /* Preserve aspect ratio and fill the unused right side with normalized + * mid-gray. Distorting every crop to target_width reduces REC accuracy. */ + for (channel = 0u; channel < 3u; ++channel) { + uint64_t index; + float padding = (float)(128.0 * normalize_scale - 1.0); + for (index = 0u; index < channel_plane; ++index) { + output[(size_t)((uint64_t)channel * channel_plane + index)] = padding; + } + } + + for (output_y = 0u; output_y < LW_REC_INPUT_HEIGHT; ++output_y) { + double source_y = ((double)output_y + 0.5) * source_height / LW_REC_INPUT_HEIGHT - 0.5; + int64_t source_y0_raw = (int64_t)floor(source_y); + int64_t source_y1_raw = source_y0_raw + 1; + uint32_t source_y0 = clamp_coordinate(source_y0_raw, source_height); + uint32_t source_y1 = clamp_coordinate(source_y1_raw, source_height); + double weight_y = source_y - (double)source_y0_raw; + uint32_t output_x; + for (output_x = 0u; output_x < actual_width; ++output_x) { + double source_x = ((double)output_x + 0.5) * source_width / actual_width - 0.5; + int64_t source_x0_raw = (int64_t)floor(source_x); + int64_t source_x1_raw = source_x0_raw + 1; + uint32_t source_x0 = clamp_coordinate(source_x0_raw, source_width); + uint32_t source_x1 = clamp_coordinate(source_x1_raw, source_width); + double weight_x = source_x - (double)source_x0_raw; + for (channel = 0u; channel < 3u; ++channel) { + double top_left = source[(size_t)((uint64_t)source_y0 * source_stride + + (uint64_t)source_x0 * 3u + channel)]; + double top_right = source[(size_t)((uint64_t)source_y0 * source_stride + + (uint64_t)source_x1 * 3u + channel)]; + double bottom_left = source[(size_t)((uint64_t)source_y1 * source_stride + + (uint64_t)source_x0 * 3u + channel)]; + double bottom_right = source[(size_t)((uint64_t)source_y1 * source_stride + + (uint64_t)source_x1 * 3u + channel)]; + double top = top_left + (top_right - top_left) * weight_x; + double bottom = bottom_left + (bottom_right - bottom_left) * weight_x; + double value = top + (bottom - top) * weight_y; + uint64_t output_index = (uint64_t)channel * channel_plane + + (uint64_t)output_y * target_width + output_x; + output[(size_t)output_index] = (float)(value * normalize_scale - 1.0); + } + } + } + if (resized_width != NULL) { + *resized_width = actual_width; + } + return LW_STATUS_OK; +} + +/* Analysis-only REC preprocessing prototype. It writes the same normalized + * BGR samples directly as NHWC ([1,H,W,C]) instead of materializing NCHW + * planes. The production recognizer continues to call the NCHW entry point + * until a complete NHWC graph segment passes end-to-end gates. */ +lw_status lw_rec_preprocess_bgr_u8_nhwc(const uint8_t* source, uint64_t source_byte_count, + uint32_t source_width, uint32_t source_height, + uint32_t source_stride, uint32_t target_width, float* output, + uint64_t output_element_count, uint32_t* resized_width) { + uint64_t row_bytes; + uint64_t required_source_bytes; + uint64_t required_output_elements; + uint64_t scaled_width_numerator; + uint64_t computed_width; + uint32_t actual_width; + uint32_t output_y; + const double normalize_scale = 2.0 / 255.0; + const float padding = (float)(128.0 * normalize_scale - 1.0); + + if (resized_width != NULL) { + *resized_width = 0u; + } + if (source == NULL || output == NULL || source_width == 0u || source_height == 0u || + target_width == 0u) { + return LW_STATUS_INVALID_ARGUMENT; + } + row_bytes = (uint64_t)source_width * 3u; + if (row_bytes > UINT32_MAX || source_stride < row_bytes || + (uint64_t)(source_height - 1u) * source_stride > UINT64_MAX - row_bytes) { + return LW_STATUS_INVALID_SHAPE; + } + required_source_bytes = (uint64_t)(source_height - 1u) * source_stride + row_bytes; + required_output_elements = (uint64_t)3u * LW_REC_INPUT_HEIGHT * target_width; + if (required_source_bytes > SIZE_MAX || source_byte_count < required_source_bytes || + output_element_count != required_output_elements || + required_output_elements > SIZE_MAX / sizeof(float)) { + return LW_STATUS_INVALID_SHAPE; + } + if ((uint64_t)source_width > (UINT64_MAX - (source_height - 1u)) / LW_REC_INPUT_HEIGHT) { + return LW_STATUS_OUT_OF_BOUNDS; + } + scaled_width_numerator = (uint64_t)LW_REC_INPUT_HEIGHT * source_width; + computed_width = (scaled_width_numerator + source_height - 1u) / source_height; + actual_width = computed_width > target_width ? target_width : (uint32_t)computed_width; + + for (output_y = 0u; output_y < LW_REC_INPUT_HEIGHT; ++output_y) { + uint32_t output_x; + for (output_x = 0u; output_x < target_width; ++output_x) { + uint32_t channel; + for (channel = 0u; channel < 3u; ++channel) { + output[(size_t)(((uint64_t)output_y * target_width + output_x) * 3u + channel)] = + padding; + } + } + } + for (output_y = 0u; output_y < LW_REC_INPUT_HEIGHT; ++output_y) { + double source_y = ((double)output_y + 0.5) * source_height / LW_REC_INPUT_HEIGHT - 0.5; + int64_t source_y0_raw = (int64_t)floor(source_y); + int64_t source_y1_raw = source_y0_raw + 1; + uint32_t source_y0 = clamp_coordinate(source_y0_raw, source_height); + uint32_t source_y1 = clamp_coordinate(source_y1_raw, source_height); + double weight_y = source_y - (double)source_y0_raw; + uint32_t output_x; + for (output_x = 0u; output_x < actual_width; ++output_x) { + double source_x = ((double)output_x + 0.5) * source_width / actual_width - 0.5; + int64_t source_x0_raw = (int64_t)floor(source_x); + int64_t source_x1_raw = source_x0_raw + 1; + uint32_t source_x0 = clamp_coordinate(source_x0_raw, source_width); + uint32_t source_x1 = clamp_coordinate(source_x1_raw, source_width); + double weight_x = source_x - (double)source_x0_raw; + uint32_t channel; + for (channel = 0u; channel < 3u; ++channel) { + double top_left = source[(size_t)((uint64_t)source_y0 * source_stride + + (uint64_t)source_x0 * 3u + channel)]; + double top_right = source[(size_t)((uint64_t)source_y0 * source_stride + + (uint64_t)source_x1 * 3u + channel)]; + double bottom_left = source[(size_t)((uint64_t)source_y1 * source_stride + + (uint64_t)source_x0 * 3u + channel)]; + double bottom_right = source[(size_t)((uint64_t)source_y1 * source_stride + + (uint64_t)source_x1 * 3u + channel)]; + double top = top_left + (top_right - top_left) * weight_x; + double bottom = bottom_left + (bottom_right - bottom_left) * weight_x; + double value = top + (bottom - top) * weight_y; + output[(size_t)(((uint64_t)output_y * target_width + output_x) * 3u + channel)] = + (float)(value * normalize_scale - 1.0); + } + } + } + if (resized_width != NULL) { + *resized_width = actual_width; + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/rec_widths_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/rec_widths_internal.h new file mode 100644 index 00000000..04f5e7bb --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/rec_widths_internal.h @@ -0,0 +1,31 @@ +#ifndef LW_REC_WIDTHS_INTERNAL_H +#define LW_REC_WIDTHS_INTERNAL_H + +#include + +/* Private width policy shared with the lazy-fallback contract. Fine buckets + * are opt-in for compiled WASM/native. Other builds keep five widths. + * Changing padding can affect attention/decoding, so native is experimental + * and requires a separate accuracy review; it is not a byte-parity kernel. */ +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_FINE_REC_WIDTHS) +#define LW_REC_RESIDENT_WIDTH_COUNT 13u +static const uint32_t lw_rec_adaptive_widths[LW_REC_RESIDENT_WIDTH_COUNT] = { + 192u, 256u, 320u, 384u, 448u, 512u, 576u, + 640u, 704u, 768u, 832u, 896u, 960u +}; +#elif !defined(__EMSCRIPTEN__) && defined(LW_NATIVE_FINE_REC_WIDTHS) +/* Preserve every production bucket. A finer policy must not increase padding + * for content that already fits 480; the WASM experiment stays independent. */ +#define LW_REC_RESIDENT_WIDTH_COUNT 14u +static const uint32_t lw_rec_adaptive_widths[LW_REC_RESIDENT_WIDTH_COUNT] = { + 192u, 256u, 320u, 384u, 448u, 480u, 512u, 576u, + 640u, 704u, 768u, 832u, 896u, 960u +}; +#else +#define LW_REC_RESIDENT_WIDTH_COUNT 5u +static const uint32_t lw_rec_adaptive_widths[LW_REC_RESIDENT_WIDTH_COUNT] = { + 192u, 320u, 480u, 640u, 960u +}; +#endif + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/recognizer.c b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/recognizer.c new file mode 100644 index 00000000..979ef570 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/recognizer.c @@ -0,0 +1,1694 @@ +#include "lw_infer.h" +#include "abi_compat_internal.h" + +/* Public REC handle: preprocessing -> graph execution -> UTF-8 CTC decoding. */ + +#include "error_internal.h" +#include "executor_internal.h" +#include "model_internal.h" +#include "profile_internal.h" +#include "rec_internal.h" +#include "rec_widths_internal.h" +#include "session_internal.h" +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +#include "x64_rec_backend_internal.h" +#include "x64_rec_fast_internal.h" +#endif + +#include +#include +#include +#include +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +#include +#endif +#if defined(_MSC_VER) +#include +#endif +#include + +#define LW_REC_DEFAULT_TARGET_WIDTH 960u +#define LW_REC_DEFAULT_MAX_IMAGE_PIXELS UINT64_C(40000000) +#define LW_MEDIUM_REC_LWM_CHECKSUM UINT64_C(0x5c1ad5136616c165) + +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +typedef struct lw_x64_rec_backend_slot { + uint32_t target_width; + lw_x64_rec_program* program; + lw_x64_rec_instance* instance; +} lw_x64_rec_backend_slot; +typedef struct lw_medium_fast_shared { + uint32_t reference_count; + lw_x64_rec_fast_plan* packed_template; +} lw_medium_fast_shared; +#endif + +typedef struct lw_rec_resident_slot { + lw_session* session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_fast_plan* fast_plan; + uint8_t fast_plan_disabled; +#endif + float* input; + float* probabilities; + uint32_t* best_indices; + float* best_probabilities; + uint64_t input_element_count; + uint64_t probability_element_count; + uint32_t target_width; + uint32_t time_steps; +} lw_rec_resident_slot; + +struct lw_recognizer { + lw_model* model; + lw_session* session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_fast_plan* fast_plan; + lw_medium_fast_shared* fast_shared; + uint8_t fast_plan_disabled; + uint8_t fast_backend_disabled; +#endif + lw_rec_dictionary* dictionary; + float* input; + float* probabilities; + uint32_t* best_indices; + float* best_probabilities; + uint64_t input_element_count; + uint64_t probability_element_count; + lw_session* cached_session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_fast_plan* cached_fast_plan; + uint8_t cached_fast_plan_disabled; +#endif + float* cached_input; + float* cached_probabilities; + uint32_t* cached_best_indices; + float* cached_best_probabilities; + uint64_t cached_input_element_count; + uint64_t cached_probability_element_count; + uint32_t cached_target_width; + uint32_t cached_time_steps; + uint64_t max_image_pixels; + lw_session_options session_options; + uint32_t current_target_width; + uint32_t current_time_steps; + uint32_t adaptive_width_enabled; + uint8_t resident_widths_enabled; + lw_rec_resident_slot resident_slots[LW_REC_RESIDENT_WIDTH_COUNT]; + /* Optional intra-op pool shared by every width slot/session of this + * recognizer. A recognizer serves one line at a time (line workers hold + * clones), so the pool is never run concurrently. */ + lw_thread_pool* intra_pool; + uint32_t intra_workers; + lw_recognizer_info info; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_backend_slot x64_slots[LW_REC_RESIDENT_WIDTH_COUNT]; + /* One shared workspace borrowed by every x64 slot instance: slots of a + * recognizer never run concurrently (one line at a time; other threads + * use clones with their own block), so the arena/scratch/CTC buffers are + * sized to the largest compiled width instead of summed over all five. */ + uint8_t* x64_shared_arena; + uint8_t* x64_shared_scratch; + float* x64_shared_ctc_scores; + uint32_t* x64_shared_best_indices; + float* x64_shared_best_probabilities; +#endif +}; + +/* Keep allocation checks width-neutral so bounded uint32_t dimensions do not + * trigger GCC's -Wtype-limits on 64-bit builds. */ +static int allocation_fits(uint64_t count, size_t element_size) { + return element_size != 0u && count <= (uint64_t)(SIZE_MAX / element_size); +} + +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +static void medium_fast_shared_release(lw_medium_fast_shared* shared) { + if (shared == NULL) return; + if (--shared->reference_count == 0u) { + lw_x64_rec_fast_plan_free(shared->packed_template); + free(shared); + } +} +#endif + +static void release_cached_session(lw_recognizer* recognizer) { +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_fast_plan_free(recognizer->cached_fast_plan); + recognizer->cached_fast_plan = NULL; + recognizer->cached_fast_plan_disabled = 0u; +#endif + free(recognizer->cached_best_probabilities); + free(recognizer->cached_best_indices); + free(recognizer->cached_probabilities); + free(recognizer->cached_input); + lw_session_free(recognizer->cached_session); + recognizer->cached_session = NULL; + recognizer->cached_input = NULL; + recognizer->cached_probabilities = NULL; + recognizer->cached_best_indices = NULL; + recognizer->cached_best_probabilities = NULL; + recognizer->cached_input_element_count = 0u; + recognizer->cached_probability_element_count = 0u; + recognizer->cached_target_width = 0u; + recognizer->cached_time_steps = 0u; +} + +static void release_resident_slot(lw_rec_resident_slot* slot) { + if (slot == NULL) { + return; + } +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_fast_plan_free(slot->fast_plan); +#endif + free(slot->best_probabilities); + free(slot->best_indices); + free(slot->probabilities); + free(slot->input); + lw_session_free(slot->session); + memset(slot, 0, sizeof(*slot)); +} + +static void swap_active_with_resident_slot(lw_recognizer* recognizer, + lw_rec_resident_slot* slot) { + lw_session* session = recognizer->session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_fast_plan* fast_plan = recognizer->fast_plan; + uint8_t fast_plan_disabled = recognizer->fast_plan_disabled; +#endif + float* input = recognizer->input; + float* probabilities = recognizer->probabilities; + uint32_t* best_indices = recognizer->best_indices; + float* best_probabilities = recognizer->best_probabilities; + uint64_t input_element_count = recognizer->input_element_count; + uint64_t probability_element_count = recognizer->probability_element_count; + uint32_t target_width = recognizer->current_target_width; + uint32_t time_steps = recognizer->current_time_steps; + + recognizer->session = slot->session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + recognizer->fast_plan = slot->fast_plan; + recognizer->fast_plan_disabled = slot->fast_plan_disabled; +#endif + recognizer->input = slot->input; + recognizer->probabilities = slot->probabilities; + recognizer->best_indices = slot->best_indices; + recognizer->best_probabilities = slot->best_probabilities; + recognizer->input_element_count = slot->input_element_count; + recognizer->probability_element_count = slot->probability_element_count; + recognizer->current_target_width = slot->target_width; + recognizer->current_time_steps = slot->time_steps; + + slot->session = session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + slot->fast_plan = fast_plan; + slot->fast_plan_disabled = fast_plan_disabled; +#endif + slot->input = input; + slot->probabilities = probabilities; + slot->best_indices = best_indices; + slot->best_probabilities = best_probabilities; + slot->input_element_count = input_element_count; + slot->probability_element_count = probability_element_count; + slot->target_width = target_width; + slot->time_steps = time_steps; +} +static void activate_cached_session(lw_recognizer* recognizer) { + lw_session* session = recognizer->session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_fast_plan* fast_plan = recognizer->fast_plan; + uint8_t fast_plan_disabled = recognizer->fast_plan_disabled; +#endif + float* input = recognizer->input; + float* probabilities = recognizer->probabilities; + uint32_t* best_indices = recognizer->best_indices; + float* best_probabilities = recognizer->best_probabilities; + uint64_t input_element_count = recognizer->input_element_count; + uint64_t probability_element_count = recognizer->probability_element_count; + uint32_t target_width = recognizer->current_target_width; + uint32_t time_steps = recognizer->current_time_steps; + + recognizer->session = recognizer->cached_session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + recognizer->fast_plan = recognizer->cached_fast_plan; + recognizer->fast_plan_disabled = recognizer->cached_fast_plan_disabled; +#endif + recognizer->input = recognizer->cached_input; + recognizer->probabilities = recognizer->cached_probabilities; + recognizer->best_indices = recognizer->cached_best_indices; + recognizer->best_probabilities = recognizer->cached_best_probabilities; + recognizer->input_element_count = recognizer->cached_input_element_count; + recognizer->probability_element_count = recognizer->cached_probability_element_count; + recognizer->current_target_width = recognizer->cached_target_width; + recognizer->current_time_steps = recognizer->cached_time_steps; + + recognizer->cached_session = session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + recognizer->cached_fast_plan = fast_plan; + recognizer->cached_fast_plan_disabled = fast_plan_disabled; +#endif + recognizer->cached_input = input; + recognizer->cached_probabilities = probabilities; + recognizer->cached_best_indices = best_indices; + recognizer->cached_best_probabilities = best_probabilities; + recognizer->cached_input_element_count = input_element_count; + recognizer->cached_probability_element_count = probability_element_count; + recognizer->cached_target_width = target_width; + recognizer->cached_time_steps = time_steps; +} + +static void clear_result(lw_recognition_result* result) { + memset(result, 0, sizeof(*result)); + result->struct_size = (uint32_t)sizeof(*result); +} + +void lw_recognizer_options_init(lw_recognizer_options* options) { + lw_model_options model_options; + lw_session_options session_options; + if (options == NULL) { + return; + } + lw_model_options_init(&model_options); + lw_session_options_init(&session_options); + memset(options, 0, sizeof(*options)); + options->struct_size = (uint32_t)sizeof(*options); + options->target_width = LW_REC_DEFAULT_TARGET_WIDTH; + options->max_model_file_size = model_options.max_file_size; + options->max_workspace_size = session_options.max_workspace_size; + options->max_tensor_size = session_options.max_tensor_size; + options->max_image_pixels = LW_REC_DEFAULT_MAX_IMAGE_PIXELS; +} + +void lw_recognizer_info_init(lw_recognizer_info* info) { + if (info == NULL) { + return; + } + memset(info, 0, sizeof(*info)); + info->struct_size = (uint32_t)sizeof(*info); +} + +void lw_recognition_result_init(lw_recognition_result* result) { + if (result == NULL) { + return; + } + clear_result(result); +} + +static lw_status validate_options(const lw_recognizer_options* options, uint32_t* target_width, + uint64_t* max_image_pixels, lw_model_options* model_options, + lw_session_options* session_options, lw_error* error) { + lw_recognizer_options defaults; + lw_recognizer_options values; + lw_recognizer_options_init(&defaults); + values = defaults; + *target_width = defaults.target_width; + *max_image_pixels = defaults.max_image_pixels; + lw_model_options_init(model_options); + lw_session_options_init(session_options); + if (options == NULL) { + return LW_STATUS_OK; + } + if (!lw_abi_copy_input_prefix(&values, sizeof(values), options)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid recognizer options structure"); + return LW_STATUS_INVALID_ARGUMENT; + } + values.struct_size = (uint32_t)sizeof(values); + if (values.reserved0 != 0u || values.reserved1 != 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid recognizer options structure"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (values.target_width != 0u) { + *target_width = values.target_width; + } + if (values.max_model_file_size != 0u) { + model_options->max_file_size = values.max_model_file_size; + } + if (values.max_workspace_size != 0u) { + session_options->max_workspace_size = values.max_workspace_size; + } + if (values.max_tensor_size != 0u) { + session_options->max_tensor_size = values.max_tensor_size; + } + if (values.max_image_pixels != 0u) { + *max_image_pixels = values.max_image_pixels; + } + if (*target_width > INT32_MAX) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "recognizer target width exceeds the tensor ABI"); + return LW_STATUS_INVALID_SHAPE; + } + return LW_STATUS_OK; +} + +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_LAZY_FALLBACK) +/* Resolve the initial output shape without allocating the canonical execution + * arena. A full session is built only if compiled coverage is incomplete or a + * compiled slot later becomes unavailable. */ +static lw_status configure_initial_metadata_session(lw_recognizer* recognizer, + uint32_t target_width, + lw_session_info* configured_info, + lw_error* error) { + lw_tensor_desc input_desc; + lw_tensor_desc output_desc; + lw_session* session = NULL; + lw_status status; + lw_tensor_desc_init(&input_desc); + input_desc.dtype = LW_DTYPE_F32; + input_desc.rank = 4u; + input_desc.dimensions[0] = 1; + input_desc.dimensions[1] = 3; + input_desc.dimensions[2] = (int32_t)LW_REC_INPUT_HEIGHT; + input_desc.dimensions[3] = (int32_t)target_width; + status = lw_session_create_metadata_only(recognizer->model, &input_desc, 1u, + &recognizer->session_options, &session, error); + if (status != LW_STATUS_OK) return status; + lw_tensor_desc_init(&output_desc); + status = lw_session_get_output_desc(session, 0u, &output_desc); + if (status != LW_STATUS_OK || output_desc.dtype != LW_DTYPE_F32 || output_desc.rank != 3u || + output_desc.dimensions[0] != 1 || output_desc.dimensions[1] <= 0 || + output_desc.dimensions[2] <= 0 || + (uint32_t)output_desc.dimensions[2] != + lw_rec_dictionary_class_count(recognizer->dictionary)) { + lw_session_free(session); + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "recognizer model output and dictionary are incompatible"); + return LW_STATUS_INVALID_SHAPE; + } + lw_session_info_init(configured_info); + status = lw_session_get_info(session, configured_info); + if (status != LW_STATUS_OK) { + lw_session_free(session); + lw_set_error(error, status, "unable to read recognizer session information"); + return status; + } + recognizer->session = session; + recognizer->current_target_width = target_width; + recognizer->current_time_steps = (uint32_t)output_desc.dimensions[1]; + return LW_STATUS_OK; +} + +static void recognizer_release_canonical_state(lw_recognizer* recognizer) { + if (recognizer == NULL) return; + release_cached_session(recognizer); + free(recognizer->best_probabilities); + free(recognizer->best_indices); + free(recognizer->probabilities); + free(recognizer->input); + recognizer->best_probabilities = NULL; + recognizer->best_indices = NULL; + recognizer->probabilities = NULL; + recognizer->input = NULL; + lw_session_free(recognizer->session); + recognizer->session = NULL; + recognizer->input_element_count = 0u; + recognizer->probability_element_count = 0u; + /* Keep current_target_width and current_time_steps as fallback metadata. */ +} +#endif + +static lw_status configure_session(lw_recognizer* recognizer, uint32_t target_width, + lw_session_info* configured_info, lw_error* error) { + lw_tensor_desc input_desc; + lw_tensor_desc output_desc; + lw_session_info session_info; + lw_session* session = NULL; + float* input = NULL; + float* probabilities = NULL; + uint32_t* best_indices = NULL; + float* best_probabilities = NULL; + uint64_t input_element_count; + uint64_t probability_element_count; + uint32_t time_steps; + uint32_t class_count; + int use_ctc_greedy; + lw_status status; + + if (recognizer->resident_widths_enabled != 0u) { + uint32_t slot_index; + if (recognizer->current_target_width == target_width) { + if (configured_info != NULL) { + lw_session_info_init(configured_info); + status = lw_session_get_info(recognizer->session, configured_info); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "unable to read resident recognizer session information"); + return status; + } + } + return LW_STATUS_OK; + } + for (slot_index = 0u; slot_index < LW_REC_RESIDENT_WIDTH_COUNT; ++slot_index) { + if (recognizer->resident_slots[slot_index].target_width == target_width) { + swap_active_with_resident_slot(recognizer, &recognizer->resident_slots[slot_index]); + if (configured_info != NULL) { + lw_session_info_init(configured_info); + status = lw_session_get_info(recognizer->session, configured_info); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "unable to read resident recognizer session information"); + return status; + } + } + return LW_STATUS_OK; + } + } + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "requested REC width is not resident"); + return LW_STATUS_INVALID_ARGUMENT; + } + + if (recognizer->cached_session != NULL && recognizer->cached_target_width == target_width) { + activate_cached_session(recognizer); + if (configured_info != NULL) { + lw_session_info_init(configured_info); + status = lw_session_get_info(recognizer->session, configured_info); + if (status != LW_STATUS_OK) { + activate_cached_session(recognizer); + lw_set_error(error, status, "unable to read recognizer session information"); + return status; + } + } + return LW_STATUS_OK; + } + + /* Keep at most two concrete widths. Discarding an inactive cache entry + * before construction bounds peak memory to the old active and candidate + * sessions; the active session remains valid if construction fails. */ + release_cached_session(recognizer); + + lw_tensor_desc_init(&input_desc); + input_desc.dtype = LW_DTYPE_F32; + input_desc.rank = 4u; + input_desc.dimensions[0] = 1; + input_desc.dimensions[1] = 3; + input_desc.dimensions[2] = (int32_t)LW_REC_INPUT_HEIGHT; + input_desc.dimensions[3] = (int32_t)target_width; + status = lw_session_create_with_prepared_source( + recognizer->model, &input_desc, 1u, &recognizer->session_options, 0u, + recognizer->session, &session, error); + if (status != LW_STATUS_OK) { + return status; + } + + lw_tensor_desc_init(&output_desc); + status = lw_session_get_output_desc(session, 0u, &output_desc); + if (status != LW_STATUS_OK || output_desc.dtype != LW_DTYPE_F32 || output_desc.rank != 3u || + output_desc.dimensions[0] != 1 || output_desc.dimensions[1] <= 0 || + output_desc.dimensions[2] <= 0 || + (uint32_t)output_desc.dimensions[2] != + lw_rec_dictionary_class_count(recognizer->dictionary)) { + lw_session_free(session); + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "recognizer model output and dictionary are incompatible"); + return LW_STATUS_INVALID_SHAPE; + } + + input_element_count = (uint64_t)3u * LW_REC_INPUT_HEIGHT * target_width; + probability_element_count = + (uint64_t)(uint32_t)output_desc.dimensions[1] * (uint32_t)output_desc.dimensions[2]; + time_steps = (uint32_t)output_desc.dimensions[1]; + class_count = (uint32_t)output_desc.dimensions[2]; + /* A terminal, last-axis Softmax can be consumed as greedy CTC classes. + * Keep the full probability buffer only for models that need the generic + * graph-output contract; official REC models use two values per step. */ + use_ctc_greedy = lw_session_supports_ctc_greedy_f32(session, time_steps, class_count); +#if defined(LW_EXPERIMENTAL_CTC_TILED) + if (use_ctc_greedy) { + /* Recreate the session with the internal physical CTC plan selected at + * construction time. The public session ABI remains generic; only + * the recognizer's private greedy path may elide terminal Softmax. */ + lw_session_free(session); + session = NULL; + status = lw_session_create_with_prepared_source( + recognizer->model, &input_desc, 1u, &recognizer->session_options, + LW_SESSION_PLAN_CTC_GREEDY, recognizer->session, &session, error); + if (status != LW_STATUS_OK) { + return status; + } + } +#endif + if (!allocation_fits(input_element_count, sizeof(*input)) || + !allocation_fits(probability_element_count, sizeof(*probabilities)) || + !allocation_fits(time_steps, sizeof(*best_indices)) || + !allocation_fits(time_steps, sizeof(*best_probabilities))) { + lw_session_free(session); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "recognizer buffer size overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + input = (float*)malloc((size_t)input_element_count * sizeof(*input)); + if (use_ctc_greedy) { + best_indices = (uint32_t*)malloc((size_t)time_steps * sizeof(*best_indices)); + best_probabilities = (float*)malloc((size_t)time_steps * sizeof(*best_probabilities)); + } else { + probabilities = (float*)malloc((size_t)probability_element_count * sizeof(*probabilities)); + } + if (input == NULL || (use_ctc_greedy && (best_indices == NULL || best_probabilities == NULL)) || + (!use_ctc_greedy && probabilities == NULL)) { + free(best_probabilities); + free(best_indices); + free(probabilities); + free(input); + lw_session_free(session); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate recognizer buffers"); + return LW_STATUS_OUT_OF_MEMORY; + } + + lw_session_info_init(&session_info); + status = lw_session_get_info(session, &session_info); + if (status != LW_STATUS_OK) { + free(best_probabilities); + free(best_indices); + free(probabilities); + free(input); + lw_session_free(session); + lw_set_error(error, status, "unable to read recognizer session information"); + return status; + } + + /* Publish the new shape only after every allocation and validation has + * succeeded. A failed adaptive switch therefore leaves the old session + * usable and avoids a partially configured recognizer. */ + recognizer->cached_session = recognizer->session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + recognizer->cached_fast_plan = recognizer->fast_plan; + recognizer->cached_fast_plan_disabled = recognizer->fast_plan_disabled; + recognizer->fast_plan = NULL; + recognizer->fast_plan_disabled = 0u; +#endif + recognizer->cached_input = recognizer->input; + recognizer->cached_probabilities = recognizer->probabilities; + recognizer->cached_best_indices = recognizer->best_indices; + recognizer->cached_best_probabilities = recognizer->best_probabilities; + recognizer->cached_input_element_count = recognizer->input_element_count; + recognizer->cached_probability_element_count = recognizer->probability_element_count; + recognizer->cached_target_width = recognizer->current_target_width; + recognizer->cached_time_steps = recognizer->current_time_steps; + recognizer->session = session; + lw_session_set_external_thread_pool(session, recognizer->intra_pool, + recognizer->intra_workers); + recognizer->input = input; + recognizer->probabilities = probabilities; + recognizer->best_indices = best_indices; + recognizer->best_probabilities = best_probabilities; + recognizer->input_element_count = input_element_count; + recognizer->probability_element_count = probability_element_count; + recognizer->current_target_width = target_width; + recognizer->current_time_steps = time_steps; + if (configured_info != NULL) { + *configured_info = session_info; + } + return LW_STATUS_OK; +} + +static uint32_t adaptive_target_width(uint32_t source_width, uint32_t source_height, + uint32_t maximum_width) { + uint64_t scaled_width; + uint32_t index; + if (source_width == 0u || source_height == 0u || maximum_width <= 320u) { + return maximum_width; + } + scaled_width = + ((uint64_t)LW_REC_INPUT_HEIGHT * source_width + source_height - 1u) / source_height; + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + if (lw_rec_adaptive_widths[index] >= maximum_width) { + break; + } + if (scaled_width <= lw_rec_adaptive_widths[index]) { + return lw_rec_adaptive_widths[index]; + } + } + return maximum_width; +} + +lw_status lw_recognizer_enable_resident_widths(lw_recognizer* recognizer, lw_error* error) { + uint32_t index; + if (recognizer == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "recognizer is required"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (recognizer->resident_widths_enabled != 0u) { + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + } + if (recognizer->info.target_width != 960u) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "resident REC widths require a 960-pixel maximum width"); + return LW_STATUS_UNSUPPORTED; + } +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + /* Fully compiled adaptive widths do not need five canonical sessions. */ + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + if (recognizer->x64_slots[index].instance == NULL) break; + } + if (index == LW_REC_RESIDENT_WIDTH_COUNT) { + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + } +#endif + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + lw_recognizer temporary = *recognizer; + lw_session_info session_info; + lw_status status; + lw_rec_resident_slot* slot = &recognizer->resident_slots[index]; + if (lw_rec_adaptive_widths[index] == recognizer->current_target_width) { + continue; + } + memset(slot, 0, sizeof(*slot)); + temporary.session = NULL; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + temporary.fast_plan = NULL; + temporary.cached_fast_plan = NULL; + temporary.fast_plan_disabled = 0u; + temporary.cached_fast_plan_disabled = 0u; +#endif + temporary.input = NULL; + temporary.probabilities = NULL; + temporary.best_indices = NULL; + temporary.best_probabilities = NULL; + temporary.input_element_count = 0u; + temporary.probability_element_count = 0u; + temporary.cached_session = NULL; + temporary.cached_input = NULL; + temporary.cached_probabilities = NULL; + temporary.cached_best_indices = NULL; + temporary.cached_best_probabilities = NULL; + temporary.cached_input_element_count = 0u; + temporary.cached_probability_element_count = 0u; + temporary.cached_target_width = 0u; + temporary.cached_time_steps = 0u; + temporary.current_target_width = 0u; + temporary.current_time_steps = 0u; + temporary.resident_widths_enabled = 0u; + /* Keep the active session as a prepared-source reference while the + * temporary recognizer constructs this resident width. The active + * session remains owned by the real recognizer; the temporary copy + * only publishes the newly created slot below. */ + temporary.session = recognizer->session; + temporary.current_target_width = recognizer->current_target_width; + lw_session_info_init(&session_info); + status = configure_session(&temporary, lw_rec_adaptive_widths[index], &session_info, error); + if (status != LW_STATUS_OK) { + uint32_t cleanup_index; + for (cleanup_index = 0u; cleanup_index < LW_REC_RESIDENT_WIDTH_COUNT; ++cleanup_index) { + release_resident_slot(&recognizer->resident_slots[cleanup_index]); + } + return status; + } + slot->session = temporary.session; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + slot->fast_plan = temporary.fast_plan; + slot->fast_plan_disabled = temporary.fast_plan_disabled; +#endif + slot->input = temporary.input; + slot->probabilities = temporary.probabilities; + slot->best_indices = temporary.best_indices; + slot->best_probabilities = temporary.best_probabilities; + slot->input_element_count = temporary.input_element_count; + slot->probability_element_count = temporary.probability_element_count; + slot->target_width = temporary.current_target_width; + slot->time_steps = temporary.current_time_steps; + } + recognizer->resident_widths_enabled = 1u; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} +lw_status lw_recognizer_enable_adaptive_width(lw_recognizer* recognizer, uint32_t enabled, + lw_error* error) { + if (recognizer == NULL || enabled > 1u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "recognizer and a boolean adaptive-width flag are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + recognizer->adaptive_width_enabled = enabled; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +uint32_t lw_recognizer_target_width_for_image(const lw_recognizer* recognizer, + uint32_t source_width, uint32_t source_height) { + if (recognizer == NULL) { + return 0u; + } + if (recognizer->adaptive_width_enabled == 0u) { + return recognizer->info.target_width; + } + return adaptive_target_width(source_width, source_height, recognizer->info.target_width); +} + +uint32_t lw_recognizer_current_target_width(const lw_recognizer* recognizer) { + return recognizer == NULL ? 0u : recognizer->current_target_width; +} + +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +static void recognizer_try_backends(lw_recognizer* recognizer); +static void recognizer_release_backends(lw_recognizer* recognizer); +#if !defined(LW_EXPERIMENTAL_CTC_TILED) +static int recognizer_has_full_compiled_coverage(const lw_recognizer* recognizer) { + uint32_t index; + int target_is_standard = 0; + if (recognizer == NULL) return 0; + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + const uint32_t width = lw_rec_adaptive_widths[index]; + const lw_x64_rec_backend_slot* slot = &recognizer->x64_slots[index]; + if (width == recognizer->info.target_width) target_is_standard = 1; + if (width <= recognizer->info.target_width && + (slot->target_width != width || slot->program == NULL || slot->instance == NULL)) + return 0; + } + return target_is_standard; +} +#endif + +static void recognizer_report_compiled_memory(const lw_recognizer* recognizer) { + uint32_t index; + uint64_t arena_bytes = 0u; + uint64_t scratch_bytes = 0u; + uint64_t ctc_rows = 0u; + uint64_t canonical_workspace = 0u; + uint64_t canonical_input = 0u; + uint64_t canonical_output = 0u; + lw_session_info session_info; + if (!lw_profile_env_is_one("LW_REC_MEMORY_PROFILE")) return; + lw_session_info_init(&session_info); + if (recognizer->session != NULL && + lw_session_get_info(recognizer->session, &session_info) == LW_STATUS_OK) { + canonical_workspace = session_info.workspace_size; + } + if (recognizer->input != NULL) + canonical_input = recognizer->input_element_count * sizeof(float); + if (recognizer->probabilities != NULL) { + canonical_output = recognizer->probability_element_count * sizeof(float); + } else { + if (recognizer->best_indices != NULL) + canonical_output += (uint64_t)recognizer->current_time_steps * sizeof(uint32_t); + if (recognizer->best_probabilities != NULL) + canonical_output += (uint64_t)recognizer->current_time_steps * sizeof(float); + } + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + const lw_x64_rec_backend_slot* slot = &recognizer->x64_slots[index]; + if (slot->program == NULL) continue; + if (slot->program->arena_bytes > arena_bytes) arena_bytes = slot->program->arena_bytes; + if (slot->program->scratch_bytes > scratch_bytes) scratch_bytes = slot->program->scratch_bytes; + if (slot->program->ctc.rows > ctc_rows) ctc_rows = slot->program->ctc.rows; + fprintf(stderr, "REC_MEMORY width=%u owned=%llu borrowed=%llu borrowed_count=%u " + "arena=%llu scratch=%llu\n", + slot->target_width, + (unsigned long long)slot->program->owned_constant_bytes, + (unsigned long long)slot->program->borrowed_constant_bytes, + slot->program->borrowed_constant_count, + (unsigned long long)slot->program->arena_bytes, + (unsigned long long)slot->program->scratch_bytes); + } + fprintf(stderr, "REC_MEMORY shared_arena=%llu shared_scratch=%llu ctc_workspace=%llu medium_fast_shared=%u\n", + (unsigned long long)(recognizer->x64_shared_arena == NULL ? 0u : arena_bytes), + (unsigned long long)(recognizer->x64_shared_scratch == NULL ? 0u : scratch_bytes), + (unsigned long long)(recognizer->x64_shared_ctc_scores == NULL ? 0u : + ctc_rows * (sizeof(float) * 2u + sizeof(uint32_t))), + recognizer->fast_shared == NULL ? 0u : 1u); + fprintf(stderr, "REC_MEMORY canonical_workspace=%llu canonical_input=%llu canonical_output=%llu canonical_planned_workspace=%llu\n", + (unsigned long long)canonical_workspace, + (unsigned long long)canonical_input, + (unsigned long long)canonical_output, + (unsigned long long)recognizer->info.workspace_size); +} +#endif + +lw_status lw_recognizer_create(const char* model_path_utf8, const char* dictionary_path_utf8, + const lw_recognizer_options* options, lw_recognizer** out_recognizer, + lw_error* error) { + lw_model_options model_options; + lw_session_options session_options; + lw_session_info session_info; + lw_recognizer* recognizer = NULL; + uint32_t target_width; + uint64_t max_image_pixels; + uint32_t max_label_bytes; + lw_status status; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_LAZY_FALLBACK) + int metadata_initial = 0; +#endif + if (out_recognizer != NULL) { + *out_recognizer = NULL; + } + if (model_path_utf8 == NULL || model_path_utf8[0] == '\0' || dictionary_path_utf8 == NULL || + dictionary_path_utf8[0] == '\0' || out_recognizer == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "model path, dictionary path, and output recognizer are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + status = validate_options(options, &target_width, &max_image_pixels, &model_options, + &session_options, error); + if (status != LW_STATUS_OK) { + return status; + } + recognizer = (lw_recognizer*)calloc(1u, sizeof(*recognizer)); + if (recognizer == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate recognizer handle"); + return LW_STATUS_OUT_OF_MEMORY; + } + recognizer->max_image_pixels = max_image_pixels; + status = lw_model_load(model_path_utf8, &model_options, &recognizer->model, error); + if (status != LW_STATUS_OK) { + goto fail; + } + status = lw_rec_dictionary_load(dictionary_path_utf8, &recognizer->dictionary, error); + if (status != LW_STATUS_OK) { + goto fail; + } + recognizer->session_options = session_options; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_LAZY_FALLBACK) + if (target_width == 960u) { + status = configure_initial_metadata_session(recognizer, target_width, + &session_info, error); + metadata_initial = 1; + } else { + status = configure_session(recognizer, target_width, &session_info, error); + } +#else + status = configure_session(recognizer, target_width, &session_info, error); +#endif + if (status != LW_STATUS_OK) { + goto fail; + } + max_label_bytes = lw_rec_dictionary_max_label_byte_count(recognizer->dictionary); + if (max_label_bytes == 0u || + (uint64_t)recognizer->current_time_steps > (UINT64_MAX - 1u) / max_label_bytes) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "recognizer maximum text capacity overflows"); + status = LW_STATUS_OUT_OF_BOUNDS; + goto fail; + } + lw_recognizer_info_init(&recognizer->info); + recognizer->info.target_width = target_width; + recognizer->info.input_height = LW_REC_INPUT_HEIGHT; + recognizer->info.time_steps = recognizer->current_time_steps; + recognizer->info.class_count = lw_rec_dictionary_class_count(recognizer->dictionary); + recognizer->info.max_text_capacity = + (uint64_t)recognizer->info.time_steps * max_label_bytes + 1u; + recognizer->info.workspace_size = session_info.workspace_size; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + recognizer_try_backends(recognizer); +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_LAZY_FALLBACK) + if (metadata_initial) { + const int full_coverage = recognizer_has_full_compiled_coverage(recognizer); + if (full_coverage) { + recognizer_release_canonical_state(recognizer); + } else { + /* Compilation is all-or-nothing for lazy retention. Rebuild the + * complete canonical fallback before publishing this handle. */ + status = configure_session(recognizer, target_width, &session_info, error); + if (status != LW_STATUS_OK) goto fail; + release_cached_session(recognizer); + recognizer->info.workspace_size = session_info.workspace_size; + } + (void)fprintf(stderr, "LW_WASM_REC_LAZY_FALLBACK full_coverage=%d canonical_retained=%u\n", + full_coverage, recognizer->session == NULL ? 0u : 1u); + } +#endif +#if !defined(LW_EXPERIMENTAL_CTC_TILED) + if (!recognizer_has_full_compiled_coverage(recognizer) && + recognizer->model->info.content_checksum == LW_MEDIUM_REC_LWM_CHECKSUM) { + lw_cpu_capabilities cpu = lw_get_cpu_capabilities(); + if (lw_simd_level_is_avx2(cpu.simd) && cpu.has_avx2_fma) { + lw_medium_fast_shared* shared = (lw_medium_fast_shared*)calloc(1u, sizeof(*shared)); + lw_error fast_error; + if (shared != NULL) { + lw_error_init(&fast_error); + if (lw_x64_rec_fast_plan_create(recognizer->session, + &shared->packed_template, + &fast_error) == LW_STATUS_OK) { + /* The template only owns immutable packed coefficients and + * folded bias; actual executions use private session plans. */ + free(shared->packed_template->nhwc_workspace); + shared->packed_template->nhwc_workspace = NULL; + shared->packed_template->nhwc_workspace_bytes = 0u; + free(shared->packed_template->scratch); + shared->packed_template->scratch = NULL; + shared->packed_template->scratch_bytes = 0u; + shared->reference_count = 1u; + recognizer->fast_shared = shared; + } else { + free(shared); + } + } + } + } +#endif + recognizer_report_compiled_memory(recognizer); +#endif + *out_recognizer = recognizer; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + +fail: + lw_recognizer_free(recognizer); + return status; +} + +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +#if defined(_MSC_VER) +static void* rec_ws_aligned_alloc(size_t size) { return _aligned_malloc(size, 64u); } +static void rec_ws_aligned_free(void* p) { _aligned_free(p); } +#else +static void* rec_ws_aligned_alloc(size_t size) { + void* p = NULL; + return posix_memalign(&p, 64u, size) == 0 ? p : NULL; +} +static void rec_ws_aligned_free(void* p) { free(p); } +#endif + +static void recognizer_release_shared_workspace(lw_recognizer* recognizer) { + rec_ws_aligned_free(recognizer->x64_shared_ctc_scores); + rec_ws_aligned_free(recognizer->x64_shared_best_indices); + rec_ws_aligned_free(recognizer->x64_shared_best_probabilities); + rec_ws_aligned_free(recognizer->x64_shared_arena); + rec_ws_aligned_free(recognizer->x64_shared_scratch); + recognizer->x64_shared_ctc_scores = NULL; + recognizer->x64_shared_best_indices = NULL; + recognizer->x64_shared_best_probabilities = NULL; + recognizer->x64_shared_arena = NULL; + recognizer->x64_shared_scratch = NULL; +} + +/* Create one instance per compiled slot, borrowing the shared workspace when + * it allocated cleanly; otherwise fall back to per-slot owned workspaces. */ +static void recognizer_create_slot_instances(lw_recognizer* recognizer) { + uint64_t arena_bytes = 0u; + uint64_t scratch_bytes = 0u; + uint64_t ctc_rows = 0u; + uint32_t index; + int ctc_needed = 0; + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + const lw_x64_rec_program* program = recognizer->x64_slots[index].program; + if (program == NULL) continue; + if (program->arena_bytes > arena_bytes) arena_bytes = program->arena_bytes; + if (program->scratch_bytes > scratch_bytes) scratch_bytes = program->scratch_bytes; + if (program->ctc.enabled != 0u) { + ctc_needed = 1; + if (program->ctc.rows > ctc_rows) ctc_rows = program->ctc.rows; + } + } + if (arena_bytes == 0u || arena_bytes > SIZE_MAX || scratch_bytes > SIZE_MAX || + ctc_rows > SIZE_MAX / sizeof(float)) { + return; + } + recognizer->x64_shared_arena = (uint8_t*)rec_ws_aligned_alloc((size_t)arena_bytes); + if (scratch_bytes > 0u) { + recognizer->x64_shared_scratch = (uint8_t*)rec_ws_aligned_alloc((size_t)scratch_bytes); + } + if (ctc_needed != 0) { + recognizer->x64_shared_ctc_scores = (float*)rec_ws_aligned_alloc((size_t)(ctc_rows * sizeof(float))); + recognizer->x64_shared_best_indices = (uint32_t*)rec_ws_aligned_alloc((size_t)(ctc_rows * sizeof(uint32_t))); + recognizer->x64_shared_best_probabilities = (float*)rec_ws_aligned_alloc((size_t)(ctc_rows * sizeof(float))); + } + if (recognizer->x64_shared_arena == NULL || + (scratch_bytes > 0u && recognizer->x64_shared_scratch == NULL) || + (ctc_needed != 0 && (recognizer->x64_shared_ctc_scores == NULL || + recognizer->x64_shared_best_indices == NULL || + recognizer->x64_shared_best_probabilities == NULL))) { + /* Shared allocation failed: per-slot owned workspaces below. */ + recognizer_release_shared_workspace(recognizer); + } + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + lw_x64_rec_backend_slot* slot = &recognizer->x64_slots[index]; + lw_error backend_error; + lw_status status; + if (slot->program == NULL) continue; + lw_error_init(&backend_error); + if (recognizer->x64_shared_arena != NULL) { + status = lw_x64_rec_instance_create_borrowed(slot->program, + recognizer->x64_shared_arena, arena_bytes, + recognizer->x64_shared_scratch, scratch_bytes, + recognizer->x64_shared_ctc_scores, recognizer->x64_shared_best_indices, + recognizer->x64_shared_best_probabilities, ctc_rows, + &slot->instance, &backend_error); + } else { + status = lw_x64_rec_instance_create(slot->program, &slot->instance, + &backend_error); + } + if (status != LW_STATUS_OK) { + lw_x64_rec_program_free(slot->program); + slot->program = NULL; + slot->instance = NULL; + } + if (slot->instance != NULL) { + lw_x64_rec_instance_set_thread_pool(slot->instance, recognizer->intra_pool, + recognizer->intra_workers); + } + } +} + +static lw_x64_rec_backend_slot* recognizer_backend_slot(lw_recognizer* recognizer, + uint32_t width) { + uint32_t index; + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + lw_x64_rec_backend_slot* slot = &recognizer->x64_slots[index]; + if (slot->target_width == width && slot->instance != NULL) return slot; + } + return NULL; +} + +/* The local rotated A/B benchmark selects NHWC for all five adaptive widths. + * A failed compile or instance allocation leaves the canonical path available. */ +static void recognizer_try_backends(lw_recognizer* recognizer) { +#if !defined(__EMSCRIPTEN__) + lw_cpu_capabilities cpu = lw_get_cpu_capabilities(); +#endif + int32_t root_index; + lw_x64_rec_program* root_program = NULL; + uint32_t index; +#if !defined(__EMSCRIPTEN__) + if (!lw_simd_level_is_avx2(cpu.simd) || !cpu.has_avx2_fma) return; +#endif + root_index = (int32_t)LW_REC_RESIDENT_WIDTH_COUNT - 1; + while (root_index >= 0 && + lw_rec_adaptive_widths[root_index] > recognizer->info.target_width) + --root_index; + if (root_index < 0) return; + { + lw_x64_rec_backend_slot* root = &recognizer->x64_slots[root_index]; + lw_error backend_error; + root->target_width = lw_rec_adaptive_widths[root_index]; + lw_error_init(&backend_error); + if (lw_x64_rec_backend_compile(recognizer->model, root->target_width, + &root->program, &backend_error) == LW_X64_REC_COMPILE_OK && + root->program != NULL && root->program->ctc.enabled != 0u && + root->program->time_steps != 0u && + root->program->class_count == recognizer->info.class_count) { + root_program = root->program; + } else { + lw_x64_rec_program_free(root->program); + root->program = NULL; + } + } + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + lw_x64_rec_backend_slot* slot = &recognizer->x64_slots[index]; + lw_error backend_error; + uint32_t width = lw_rec_adaptive_widths[index]; + if (width > recognizer->info.target_width || (int32_t)index == root_index) continue; + slot->target_width = width; + lw_error_init(&backend_error); + if (root_program == NULL || + lw_x64_rec_backend_compile_shared(recognizer->model, width, root_program, + &slot->program, &backend_error) != LW_X64_REC_COMPILE_OK) { + lw_x64_rec_program_free(slot->program); + slot->program = NULL; + lw_error_init(&backend_error); + (void)lw_x64_rec_backend_compile(recognizer->model, width, + &slot->program, &backend_error); + } + if (slot->program == NULL || slot->program->ctc.enabled == 0u || + slot->program->time_steps == 0u || + slot->program->class_count != recognizer->info.class_count) { + lw_x64_rec_program_free(slot->program); + slot->program = NULL; + slot->instance = NULL; + } + } + recognizer_create_slot_instances(recognizer); +#if defined(__EMSCRIPTEN__) + { + uint32_t compiled = 0u; + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + if (recognizer->x64_slots[index].instance != NULL) ++compiled; + } + fprintf(stderr, "LW_WASM_COMPILED_REC widths=%u/%u\n", + compiled, LW_REC_RESIDENT_WIDTH_COUNT); + } +#endif +} + +static void recognizer_release_backends(lw_recognizer* recognizer) { + uint32_t index; + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + lw_x64_rec_backend_slot* slot = &recognizer->x64_slots[index]; + lw_x64_rec_instance_free(slot->instance); + lw_x64_rec_program_free(slot->program); + memset(slot, 0, sizeof(*slot)); + } + recognizer_release_shared_workspace(recognizer); +} + +/* The hybrid plan supports this exact Medium LWM graph across the five + * adaptive widths. Small has known argmax mismatches and must stay on the + * canonical executor until its numerical contract is fixed. */ +static int recognizer_try_medium_fast_plan(lw_recognizer* recognizer) { +#if defined(LW_EXPERIMENTAL_CTC_TILED) + (void)recognizer; + return 0; +#else + lw_cpu_capabilities cpu; + lw_error backend_error; + uint32_t width_index; + int owns_probabilities = 0; + if (recognizer->model->info.content_checksum != LW_MEDIUM_REC_LWM_CHECKSUM || + recognizer->fast_shared == NULL || + recognizer->fast_backend_disabled != 0u || + recognizer->fast_plan_disabled != 0u) { + return 0; + } + cpu = lw_get_cpu_capabilities(); + if (!lw_simd_level_is_avx2(cpu.simd) || !cpu.has_avx2_fma) return 0; + for (width_index = 0u; width_index < LW_REC_RESIDENT_WIDTH_COUNT; ++width_index) { + if (recognizer->current_target_width == lw_rec_adaptive_widths[width_index]) break; + } + if (width_index == LW_REC_RESIDENT_WIDTH_COUNT) return 0; + if (recognizer->probabilities == NULL) { + if (!allocation_fits(recognizer->probability_element_count, sizeof(float))) { + recognizer->fast_plan_disabled = 1u; + return 0; + } + recognizer->probabilities = (float*)malloc( + (size_t)recognizer->probability_element_count * sizeof(float)); + if (recognizer->probabilities == NULL) { + recognizer->fast_plan_disabled = 1u; + return 0; + } + owns_probabilities = 1; + } + if (recognizer->fast_plan == NULL) { + lw_error_init(&backend_error); + if (lw_x64_rec_fast_plan_create_shared(recognizer->session, + recognizer->fast_shared->packed_template, + &recognizer->fast_plan, &backend_error) != LW_STATUS_OK) { + if (owns_probabilities != 0) { + free(recognizer->probabilities); + recognizer->probabilities = NULL; + } + recognizer->fast_plan_disabled = 1u; + return 0; + } + } + return 1; +#endif +} +#endif + +lw_status lw_recognizer_clone(const lw_recognizer* source, lw_recognizer** out_recognizer, + lw_error* error) { + lw_recognizer* clone; + lw_session_info session_info; + lw_status status; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_LAZY_FALLBACK) + int metadata_clone = 0; +#endif + if (out_recognizer != NULL) { + *out_recognizer = NULL; + } + if (source == NULL || out_recognizer == NULL || source->model == NULL || + source->dictionary == NULL || source->current_target_width == 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "source recognizer and output handle are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + clone = (lw_recognizer*)calloc(1u, sizeof(*clone)); + if (clone == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate recognizer clone"); + return LW_STATUS_OUT_OF_MEMORY; + } + clone->model = source->model; + clone->dictionary = source->dictionary; + clone->max_image_pixels = source->max_image_pixels; + clone->session_options = source->session_options; + clone->adaptive_width_enabled = source->adaptive_width_enabled; + clone->info = source->info; + lw_model_retain(clone->model); + lw_rec_dictionary_retain(clone->dictionary); + lw_session_info_init(&session_info); +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_LAZY_FALLBACK) + if (source->session == NULL && source->info.target_width == 960u) { + status = configure_initial_metadata_session(clone, source->current_target_width, + &session_info, error); + metadata_clone = 1; + } else { + status = configure_session(clone, source->current_target_width, &session_info, error); + } +#else + status = configure_session(clone, source->current_target_width, &session_info, error); +#endif + if (status != LW_STATUS_OK) { + lw_recognizer_free(clone); + return status; + } +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + clone->fast_backend_disabled = source->fast_backend_disabled; + clone->fast_plan_disabled = source->fast_plan_disabled; + clone->fast_shared = source->fast_shared; + if (clone->fast_shared != NULL) ++clone->fast_shared->reference_count; +#endif + if (source->session != NULL) { + status = lw_session_share_prepared_constants(clone->session, source->session, error); + if (status != LW_STATUS_OK) { + lw_recognizer_free(clone); + return status; + } + } + clone->info.time_steps = clone->current_time_steps; + clone->info.workspace_size = session_info.workspace_size; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + { + uint32_t index; + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + const lw_x64_rec_backend_slot* source_slot = &source->x64_slots[index]; + lw_x64_rec_backend_slot* clone_slot = &clone->x64_slots[index]; + if (source_slot->instance == NULL) continue; + clone_slot->target_width = source_slot->target_width; + clone_slot->program = source_slot->program; + lw_x64_rec_program_retain(clone_slot->program); + } + recognizer_create_slot_instances(clone); + } +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_LAZY_FALLBACK) + if (metadata_clone) { + if (recognizer_has_full_compiled_coverage(clone)) { + recognizer_release_canonical_state(clone); + } else { + status = configure_session(clone, source->current_target_width, + &session_info, error); + if (status != LW_STATUS_OK) { + lw_recognizer_free(clone); + return status; + } + release_cached_session(clone); + clone->info.workspace_size = session_info.workspace_size; + } + } +#endif +#endif + lw_recognizer_set_intra_op_thread_count(clone, source->intra_workers); + *out_recognizer = clone; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) +/* Test-only internal hook: disable the compiled x64 REC backend so the same + * binary can measure canonical recognizer timing. */ +void lw_recognizer_test_disable_x64_backend(lw_recognizer* recognizer) { + uint32_t index; + if (recognizer == NULL) return; + recognizer_release_backends(recognizer); + recognizer->fast_backend_disabled = 1u; + lw_x64_rec_fast_plan_free(recognizer->fast_plan); + recognizer->fast_plan = NULL; + recognizer->fast_plan_disabled = 1u; + lw_x64_rec_fast_plan_free(recognizer->cached_fast_plan); + recognizer->cached_fast_plan = NULL; + recognizer->cached_fast_plan_disabled = 1u; + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + lw_rec_resident_slot* slot = &recognizer->resident_slots[index]; + lw_x64_rec_fast_plan_free(slot->fast_plan); + slot->fast_plan = NULL; + slot->fast_plan_disabled = 1u; + } + medium_fast_shared_release(recognizer->fast_shared); + recognizer->fast_shared = NULL; +} + +int lw_recognizer_test_has_canonical_session(const lw_recognizer* recognizer) { + return recognizer != NULL && recognizer->session != NULL; +} + +uint32_t lw_recognizer_test_compiled_width_count(const lw_recognizer* recognizer) { + uint32_t count = 0u; + if (recognizer == NULL) return 0u; + for (uint32_t index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) + if (recognizer->x64_slots[index].instance != NULL) ++count; + return count; +} +#endif + +void lw_recognizer_free(lw_recognizer* recognizer) { + if (recognizer == NULL) { + return; + } + free(recognizer->best_probabilities); + free(recognizer->best_indices); + free(recognizer->probabilities); + free(recognizer->input); +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_fast_plan_free(recognizer->fast_plan); + recognizer_release_backends(recognizer); +#endif + lw_session_free(recognizer->session); + { + uint32_t slot_index; + for (slot_index = 0u; slot_index < LW_REC_RESIDENT_WIDTH_COUNT; ++slot_index) { + release_resident_slot(&recognizer->resident_slots[slot_index]); + } + } + release_cached_session(recognizer); +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + medium_fast_shared_release(recognizer->fast_shared); +#endif + lw_thread_pool_free(recognizer->intra_pool); + recognizer->intra_pool = NULL; + lw_rec_dictionary_free(recognizer->dictionary); + lw_model_free(recognizer->model); + free(recognizer); +} + +static void recognizer_attach_intra_pool(lw_recognizer* recognizer) { + uint32_t index; + if (recognizer->session != NULL) { + lw_session_set_external_thread_pool(recognizer->session, recognizer->intra_pool, + recognizer->intra_workers); + } + if (recognizer->cached_session != NULL) { + lw_session_set_external_thread_pool(recognizer->cached_session, recognizer->intra_pool, + recognizer->intra_workers); + } + for (index = 0u; index < LW_REC_RESIDENT_WIDTH_COUNT; ++index) { + if (recognizer->resident_slots[index].session != NULL) { + lw_session_set_external_thread_pool(recognizer->resident_slots[index].session, + recognizer->intra_pool, recognizer->intra_workers); + } +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + if (recognizer->x64_slots[index].instance != NULL) { + lw_x64_rec_instance_set_thread_pool(recognizer->x64_slots[index].instance, + recognizer->intra_pool, recognizer->intra_workers); + } +#endif + } +} + +void lw_recognizer_set_intra_op_thread_count(lw_recognizer* recognizer, uint32_t thread_count) { + lw_thread_pool* replacement; + uint32_t capacity; + if (recognizer == NULL) return; + if (thread_count == 0u) thread_count = 1u; + if (thread_count > LW_PARALLEL_MAX_WORKERS) thread_count = LW_PARALLEL_MAX_WORKERS; + capacity = lw_thread_pool_worker_count(recognizer->intra_pool); + if (thread_count == recognizer->intra_workers && + (thread_count == 1u || capacity >= thread_count)) return; + if (thread_count > 1u && capacity < thread_count) { + replacement = lw_thread_pool_create(thread_count); + if (replacement != NULL) { + uint32_t replacement_capacity = lw_thread_pool_worker_count(replacement); + if (replacement_capacity > capacity) { + lw_thread_pool_free(recognizer->intra_pool); + recognizer->intra_pool = replacement; + capacity = replacement_capacity; + } else { + lw_thread_pool_free(replacement); + } + } + } + /* Keep the largest pool for later sparse requests. A smaller active count + * is passed to each kernel, so parked threads do no work or allocation. */ + recognizer->intra_workers = thread_count < capacity ? thread_count : capacity; + if (recognizer->intra_workers == 0u) recognizer->intra_workers = 1u; + recognizer_attach_intra_pool(recognizer); +} + +lw_status lw_recognizer_get_info(const lw_recognizer* recognizer, lw_recognizer_info* info) { + if (recognizer == NULL || info == NULL || + !lw_abi_copy_output_prefix(info, info->struct_size, &recognizer->info, + sizeof(recognizer->info))) { + return LW_STATUS_INVALID_ARGUMENT; + } + return LW_STATUS_OK; +} + +static lw_status recognizer_recognize_bgr_u8_impl(lw_recognizer* recognizer, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + char* text_utf8, uint64_t text_capacity, + lw_recognition_result* result, + lw_pipeline_component_profile* profile, + lw_error* error) { + uint64_t source_pixels; + uint32_t resized_width = 0u; + uint64_t required_capacity = 0u; + float score = 0.0f; + uint32_t emitted_count = 0u; + uint32_t target_width; + uint32_t execution_time_steps; + uint32_t width_bucket = 0u; + uint64_t node_nanoseconds_before[LW_EXECUTION_PROFILE_NODE_CAPACITY]; + uint64_t node_invocations_before[LW_EXECUTION_PROFILE_NODE_CAPACITY]; + lw_recognition_result output; + uint32_t result_size; + uint64_t started; + lw_status status; + uint32_t* execution_best_indices = NULL; + float* execution_best_probabilities = NULL; + int backend_active = 0; + int hybrid_active = 0; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + lw_x64_rec_backend_slot* backend_slot = NULL; + float* backend_input = NULL; + uint64_t backend_input_count = 0u; +#endif + if (recognizer == NULL || source == NULL || result == NULL || + result->struct_size < sizeof(uint32_t) || (text_utf8 == NULL && text_capacity != 0u)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "recognizer, BGR source, and initialized result are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + result_size = result->struct_size; + clear_result(&output); + if (source_width == 0u || source_height == 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "source image dimensions must be positive"); + return LW_STATUS_INVALID_ARGUMENT; + } + source_pixels = (uint64_t)source_width * source_height; + if (source_pixels > recognizer->max_image_pixels) { + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, "source image exceeds max_image_pixels"); + return LW_STATUS_MEMORY_LIMIT; + } + target_width = lw_recognizer_target_width_for_image(recognizer, source_width, source_height); +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + backend_slot = recognizer_backend_slot(recognizer, target_width); + backend_active = backend_slot != NULL; +#endif + if (profile != NULL) { + if (backend_active || recognizer->resident_widths_enabled != 0u || + (recognizer->session != NULL && + target_width == recognizer->current_target_width) || + target_width == recognizer->cached_target_width) { + if (profile->session_cache_hits != UINT64_MAX) { + ++profile->session_cache_hits; + } + } else { + if (profile->session_cache_misses != UINT64_MAX) { + ++profile->session_cache_misses; + } + if (profile->session_reconfigurations != UINT64_MAX) { + ++profile->session_reconfigurations; + } + } + } + if (!backend_active && (recognizer->session == NULL || + target_width != recognizer->current_target_width)) { + status = configure_session(recognizer, target_width, NULL, error); + if (status != LW_STATUS_OK) { + return status; + } + } + execution_time_steps = recognizer->current_time_steps; +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + if (backend_active) execution_time_steps = backend_slot->program->time_steps; +#endif + started = lw_pipeline_profile_now(profile); +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + if (backend_active) { + backend_input = lw_x64_rec_instance_input(backend_slot->instance, &backend_input_count); + if (backend_slot->program->backend_layout == LW_X64_REC_BACKEND_NCHW) { + status = lw_rec_preprocess_bgr_u8(source, source_byte_count, source_width, + source_height, source_stride, target_width, backend_input, + backend_input_count, &resized_width); + } else { + status = lw_rec_preprocess_bgr_u8_nhwc(source, source_byte_count, source_width, + source_height, source_stride, target_width, + backend_input, backend_input_count, + &resized_width); + } + } else +#endif + { + status = lw_rec_preprocess_bgr_u8(source, source_byte_count, source_width, source_height, + source_stride, recognizer->current_target_width, recognizer->input, + recognizer->input_element_count, &resized_width); + } +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + if (status != LW_STATUS_OK && backend_active) { + backend_active = 0; + if (recognizer->session == NULL || target_width != recognizer->current_target_width) { + status = configure_session(recognizer, target_width, NULL, error); + if (status != LW_STATUS_OK) return status; + } + execution_time_steps = recognizer->current_time_steps; + status = lw_rec_preprocess_bgr_u8(source, source_byte_count, source_width, source_height, + source_stride, recognizer->current_target_width, recognizer->input, + recognizer->input_element_count, &resized_width); + } +#endif + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "BGR source layout is invalid"); + return status; + } +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + if (!backend_active) { + hybrid_active = recognizer_try_medium_fast_plan(recognizer); + } +#endif + lw_pipeline_profile_add_elapsed(profile == NULL ? NULL : &profile->preprocess_nanoseconds, + started, profile); + if (profile != NULL) { + width_bucket = lw_rec_width_histogram_bucket(resized_width); + memcpy(node_nanoseconds_before, profile->execution.node_nanoseconds, + sizeof(node_nanoseconds_before)); + memcpy(node_invocations_before, profile->execution.node_invocations, + sizeof(node_invocations_before)); + } started = lw_pipeline_profile_now(profile); +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + if (backend_active && status == LW_STATUS_OK) { + status = lw_x64_rec_instance_run(backend_slot->instance, error); + if (status == LW_STATUS_OK) { + execution_best_indices = backend_slot->instance->best_indices; + execution_best_probabilities = backend_slot->instance->best_probabilities; + } else { +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_LAZY_FALLBACK) + /* Retire the failed private instance. Rebuild a full NCHW + * session and preprocess again; the backend input is NHWC. */ + lw_x64_rec_instance_free(backend_slot->instance); + backend_slot->instance = NULL; + backend_active = 0; + lw_error_init(error); + if (recognizer->session == NULL || + target_width != recognizer->current_target_width) { + status = configure_session(recognizer, target_width, NULL, error); + if (status != LW_STATUS_OK) return status; + } + execution_time_steps = recognizer->current_time_steps; + status = lw_rec_preprocess_bgr_u8( + source, source_byte_count, source_width, source_height, source_stride, + recognizer->current_target_width, recognizer->input, + recognizer->input_element_count, &resized_width); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "BGR source layout is invalid"); + return status; + } +#else + /* The recognizer input contains the backend layout after a successful + * backend preprocess. Do not silently feed that NHWC buffer to the + * canonical NCHW executor after a runtime backend failure. */ + return status; +#endif + } + } +#endif + if (!backend_active) { +#if defined(LW_EXPERIMENTAL_AVX2_FAST_PATH) + if (hybrid_active) { + status = lw_x64_rec_fast_run( + recognizer->fast_plan, recognizer->input, recognizer->input_element_count, + recognizer->probabilities, recognizer->probability_element_count, error); + if (status != LW_STATUS_OK) { + lw_x64_rec_fast_plan_free(recognizer->fast_plan); + recognizer->fast_plan = NULL; + recognizer->fast_plan_disabled = 1u; + hybrid_active = 0; + lw_error_init(error); + } + } +#endif + if (!hybrid_active && recognizer->best_indices != NULL) { + status = lw_execute_session_f32_ctc_greedy( + recognizer->session, recognizer->input, recognizer->input_element_count, + recognizer->best_indices, recognizer->best_probabilities, + execution_time_steps, recognizer->info.class_count, + profile == NULL ? NULL : &profile->execution, error); + if (status == LW_STATUS_OK) { + execution_best_indices = recognizer->best_indices; + execution_best_probabilities = recognizer->best_probabilities; + } + } else if (!hybrid_active) { + status = profile == NULL + ? lw_execute_session_f32( + recognizer->session, recognizer->input, + recognizer->input_element_count, recognizer->probabilities, + recognizer->probability_element_count, error) + : lw_execute_session_f32_profiled( + recognizer->session, recognizer->input, + recognizer->input_element_count, recognizer->probabilities, + recognizer->probability_element_count, &profile->execution, error); + } + } + if (status == LW_STATUS_OK && profile != NULL) { + lw_pipeline_profile_capture_node_width_delta( + profile, width_bucket, node_nanoseconds_before, node_invocations_before); + } + lw_pipeline_profile_add_elapsed(profile == NULL ? NULL : &profile->graph_nanoseconds, started, + profile); + if (status != LW_STATUS_OK) { + return status; + } + if (profile != NULL) { + uint64_t* count = (backend_active || hybrid_active) + ? &profile->compiled_backend_lines + : &profile->canonical_fallback_lines; + if (*count != UINT64_MAX) ++*count; + } + started = lw_pipeline_profile_now(profile); + if (execution_best_indices != NULL) { + if (text_utf8 != NULL && text_capacity >= recognizer->info.max_text_capacity) { + status = lw_rec_ctc_decode_greedy_known_capacity_f32( + recognizer->dictionary, execution_best_indices, + execution_best_probabilities, execution_time_steps, + recognizer->info.class_count, text_utf8, text_capacity, &required_capacity, + &score, &emitted_count, error); + } else { + status = lw_rec_ctc_decode_greedy_f32( + recognizer->dictionary, execution_best_indices, + execution_best_probabilities, execution_time_steps, + recognizer->info.class_count, text_utf8, text_capacity, &required_capacity, + &score, &emitted_count, error); + } + } else if (text_utf8 != NULL && text_capacity >= recognizer->info.max_text_capacity) { + status = lw_rec_ctc_decode_known_capacity_f32( + recognizer->dictionary, recognizer->probabilities, + recognizer->probability_element_count, execution_time_steps, + recognizer->info.class_count, text_utf8, text_capacity, &required_capacity, &score, + &emitted_count, error); + } else { + status = lw_rec_ctc_decode_f32(recognizer->dictionary, recognizer->probabilities, + recognizer->probability_element_count, + execution_time_steps, recognizer->info.class_count, + text_utf8, text_capacity, &required_capacity, &score, + &emitted_count, error); + } + output.emitted_count = emitted_count; + output.score = score; + output.resized_width = resized_width; + output.time_steps = execution_time_steps; + output.required_text_capacity = required_capacity; + lw_pipeline_profile_add_elapsed(profile == NULL ? NULL : &profile->postprocess_nanoseconds, + started, profile); + (void)lw_abi_copy_output_prefix(result, result_size, &output, sizeof(output)); + return status; +} + +lw_status lw_recognizer_recognize_bgr_u8(lw_recognizer* recognizer, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + char* text_utf8, uint64_t text_capacity, + lw_recognition_result* result, lw_error* error) { + return recognizer_recognize_bgr_u8_impl(recognizer, source, source_byte_count, source_width, + source_height, source_stride, text_utf8, text_capacity, + result, NULL, error); +} + +lw_status lw_recognizer_recognize_bgr_u8_profiled(lw_recognizer* recognizer, const uint8_t* source, + uint64_t source_byte_count, uint32_t source_width, + uint32_t source_height, uint32_t source_stride, + char* text_utf8, uint64_t text_capacity, + lw_recognition_result* result, + lw_pipeline_component_profile* profile, + lw_error* error) { + if (profile == NULL || profile->execution.struct_size != sizeof(profile->execution) || + profile->execution.clock == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "an initialized recognizer profile and clock are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + return recognizer_recognize_bgr_u8_impl(recognizer, source, source_byte_count, source_width, + source_height, source_stride, text_utf8, text_capacity, + result, profile, error); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/resize_fixed_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/resize_fixed_internal.h new file mode 100644 index 00000000..28a748fa --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/ppocr/resize_fixed_internal.h @@ -0,0 +1,101 @@ +#ifndef LW_RESIZE_FIXED_INTERNAL_H +#define LW_RESIZE_FIXED_INTERNAL_H + +/* Shared 11-bit fixed-point OpenCV INTER_LINEAR replica used by the DET and + * CLS preprocessing paths. The coordinate math stays in double (OpenCV + * softdouble parity) and the coefficients use round-half-even + * (CvRoundToShort). */ + +#include +#include +#include + +#define LW_FIXED_RESIZE_SCALE 2048 + +static int16_t lw_fixed_round_to_short(double value) { + double rounded = floor(value + 0.5); + if (rounded - value == 0.5 && ((int64_t)rounded & 1) != 0) { + rounded -= 1.0; + } + return (int16_t)rounded; +} + +/* Per-destination-column coefficients with edge clamping: both source + * indices clamp, and the fraction zeroes at the edges. */ +static void lw_fixed_build_linear_coefficients(uint32_t source_size, uint32_t destination_size, + int32_t* offsets, int16_t* coefficients) { + uint32_t d; + for (d = 0u; d < destination_size; ++d) { + double coordinate = ((double)d + 0.5) * source_size / destination_size - 0.5; + int32_t source_index = (int32_t)floor(coordinate); + double fraction = coordinate - source_index; + if (source_index < 0) { + source_index = 0; + fraction = 0.0; + } + if (source_index >= (int32_t)source_size - 1) { + source_index = (int32_t)source_size - 1; + fraction = 0.0; + } + offsets[d] = source_index; + coefficients[(size_t)d * 2u] = + lw_fixed_round_to_short((1.0 - fraction) * LW_FIXED_RESIZE_SCALE); + coefficients[(size_t)d * 2u + 1u] = + lw_fixed_round_to_short(fraction * LW_FIXED_RESIZE_SCALE); + } +} + +/* Vertical coordinate: raw floor/fraction without edge clamping (both source + * rows point at the edge pixel but the two fixed-point products are still + * evaluated separately). */ +static void lw_fixed_get_linear_coordinate(uint32_t destination, uint32_t source_size, + uint32_t destination_size, int32_t* source_index, + int16_t* coefficient0, int16_t* coefficient1) { + double coordinate = ((double)destination + 0.5) * source_size / destination_size - 0.5; + *source_index = (int32_t)floor(coordinate); + { + double fraction = coordinate - *source_index; + *coefficient0 = lw_fixed_round_to_short((1.0 - fraction) * LW_FIXED_RESIZE_SCALE); + *coefficient1 = lw_fixed_round_to_short(fraction * LW_FIXED_RESIZE_SCALE); + } +} + +/* One horizontal pass into int32 row scratch (BGR24 source). */ +static void lw_fixed_build_horizontal_row(const uint8_t* source, uint32_t source_stride, + uint32_t source_width, uint32_t source_y, + uint32_t destination_width, const int32_t* offsets, + const int16_t* coefficients, int32_t* destination) { + const uint8_t* row = source + (size_t)source_y * source_stride; + uint32_t x; + for (x = 0u; x < destination_width; ++x) { + int32_t sx = offsets[x]; + int32_t sx1 = sx + 1; + int16_t coefficient0 = coefficients[(size_t)x * 2u]; + int16_t coefficient1 = coefficients[(size_t)x * 2u + 1u]; + if (sx1 >= (int32_t)source_width) sx1 = (int32_t)source_width - 1; + { + size_t source_offset = (size_t)sx * 3u; + size_t source_offset1 = (size_t)sx1 * 3u; + size_t destination_offset = (size_t)x * 3u; + destination[destination_offset] = (int32_t)row[source_offset] * coefficient0 + + (int32_t)row[source_offset1] * coefficient1; + destination[destination_offset + 1u] = + (int32_t)row[source_offset + 1u] * coefficient0 + + (int32_t)row[source_offset1 + 1u] * coefficient1; + destination[destination_offset + 2u] = + (int32_t)row[source_offset + 2u] * coefficient0 + + (int32_t)row[source_offset1 + 2u] * coefficient1; + } + } +} + +/* VResizeLinearVec_32s8u replica: vertical fixed-point blend of two + * horizontal accumulators, clamped to [0, 255]. */ +static int32_t lw_fixed_blend(int32_t h0, int32_t h1, int16_t beta0, int16_t beta1) { + int32_t value = (((h0 >> 4) * beta0 >> 16) + ((h1 >> 4) * beta1 >> 16) + 2) >> 2; + if (value < 0) value = 0; + if (value > 255) value = 255; + return value; +} + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/abi_compat_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/abi_compat_internal.h new file mode 100644 index 00000000..51650f35 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/abi_compat_internal.h @@ -0,0 +1,74 @@ +#ifndef LW_ABI_COMPAT_INTERNAL_H +#define LW_ABI_COMPAT_INTERNAL_H + +/* + * Helpers for additive ABI evolution. A caller advertises the number of + * bytes available through the leading struct_size field. Output APIs copy + * only the common prefix, ignore a newer caller's unknown tail, and preserve + * the caller's advertised size in the returned structure. + */ + +#include +#include +#include + +static inline int lw_abi_copy_output_prefix(void* destination, uint32_t supplied_size, + const void* source, size_t current_size) { + size_t copy_size; + if (destination == NULL || source == NULL || supplied_size < sizeof(uint32_t)) { + return 0; + } + copy_size = (size_t)supplied_size; + if (copy_size > current_size) { + copy_size = current_size; + } + memcpy(destination, source, copy_size); + memcpy(destination, &supplied_size, sizeof(supplied_size)); + return 1; +} + +static inline int lw_abi_copy_input_prefix(void* destination, size_t current_size, + const void* source) { + uint32_t supplied_size; + size_t copy_size; + if (destination == NULL || source == NULL) { + return 0; + } + memcpy(&supplied_size, source, sizeof(supplied_size)); + if (supplied_size < sizeof(uint32_t)) { + return 0; + } + copy_size = (size_t)supplied_size; + if (copy_size > current_size) { + copy_size = current_size; + } + memcpy(destination, source, copy_size); + return 1; +} + +static inline int lw_abi_copy_nested_input_prefix(void* destination, size_t current_size, + const void* source, size_t available_size) { + uint32_t supplied_size; + size_t copy_size; + if (destination == NULL || source == NULL) { + return 0; + } + if (available_size < sizeof(uint32_t)) { + return 1; + } + memcpy(&supplied_size, source, sizeof(supplied_size)); + if (supplied_size < sizeof(uint32_t)) { + return 0; + } + copy_size = (size_t)supplied_size; + if (copy_size > available_size) { + copy_size = available_size; + } + if (copy_size > current_size) { + copy_size = current_size; + } + memcpy(destination, source, copy_size); + return 1; +} + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/atomic_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/atomic_internal.h new file mode 100644 index 00000000..28e6c429 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/atomic_internal.h @@ -0,0 +1,50 @@ +#ifndef LW_ATOMIC_INTERNAL_H +#define LW_ATOMIC_INTERNAL_H + +/* Small internal-only acquire/release counter used by shared prepared data. */ + +#include +#include + +#if defined(_MSC_VER) +# include +#endif + +typedef volatile uint32_t lw_atomic_u32; + +static inline void lw_atomic_u32_init(lw_atomic_u32* value, uint32_t initial) { + if (value != NULL) { + *value = initial; + } +} + +static inline uint32_t lw_atomic_u32_load_acquire(const lw_atomic_u32* value) { +#if defined(_MSC_VER) + return (uint32_t)_InterlockedCompareExchange((volatile long*)value, 0L, 0L); +#elif defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(value, __ATOMIC_ACQUIRE); +#else +# error "No atomic implementation for this compiler." +#endif +} + +static inline int lw_atomic_u32_compare_exchange_acq_rel(lw_atomic_u32* value, + uint32_t* expected, + uint32_t desired) { +#if defined(_MSC_VER) + const long observed = _InterlockedCompareExchange((volatile long*)value, + (long)desired, (long)*expected); + if ((uint32_t)observed == *expected) { + return 1; + } + *expected = (uint32_t)observed; + return 0; +#elif defined(__GNUC__) || defined(__clang__) + return __atomic_compare_exchange_n(value, expected, desired, 0, __ATOMIC_ACQ_REL, + __ATOMIC_ACQUIRE); +#else +# error "No atomic implementation for this compiler." +#endif +} + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/cpu_topology.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/cpu_topology.c new file mode 100644 index 00000000..db703823 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/cpu_topology.c @@ -0,0 +1,306 @@ +#if defined(__linux__) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE +#endif + +#include "cpu_topology.h" +#include "parallel_internal.h" + +/* CPU discovery is deliberately dependency-free and best effort. Failure to + * inspect topology must reduce parallelism conservatively, never prevent OCR. */ + +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# if !defined(_WIN32_WINNT) +# define _WIN32_WINNT 0x0601 +# endif +# define WIN32_LEAN_AND_MEAN +# include +#elif defined(__APPLE__) +# include +#elif !defined(__EMSCRIPTEN__) +# include +# if defined(__linux__) +# include +# endif +#endif + +#define LW_DEFAULT_NATIVE_PARALLEL_CAP 8u +#define LW_DEFAULT_NATIVE_PROCESSOR_FALLBACK 4u + +#if !defined(_WIN32) && !defined(__APPLE__) && !defined(__EMSCRIPTEN__) +static uint32_t clamp_positive_u64(uint64_t value) { + if (value == 0u) { + return 1u; + } + return value > UINT32_MAX ? UINT32_MAX : (uint32_t)value; +} +#endif + +#if defined(_WIN32) +static uint32_t count_affinity_bits(DWORD_PTR mask) { + uint32_t count = 0u; + while (mask != 0u) { + count += (uint32_t)(mask & 1u); + mask >>= 1u; + } + return count; +} + +static uint32_t windows_physical_core_count(void) { + DWORD bytes = 0u; + uint8_t* buffer; + uint8_t* cursor; + uint8_t* end; + uint32_t count = 0u; + if (GetLogicalProcessorInformationEx(RelationProcessorCore, NULL, &bytes) != FALSE || + GetLastError() != ERROR_INSUFFICIENT_BUFFER || bytes == 0u) { + return 0u; + } + buffer = (uint8_t*)malloc(bytes); + if (buffer == NULL) { + return 0u; + } + if (!GetLogicalProcessorInformationEx( + RelationProcessorCore, (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)buffer, &bytes)) { + free(buffer); + return 0u; + } + cursor = buffer; + end = buffer + bytes; + while ((size_t)(end - cursor) >= sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)) { + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX item = + (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)(void*)cursor; + if (item->Size < sizeof(*item) || (size_t)(end - cursor) < item->Size) { + break; + } + if (item->Relationship == RelationProcessorCore && count != UINT32_MAX) { + ++count; + } + cursor += item->Size; + } + free(buffer); + return count; +} +#endif + +#if defined(__linux__) && !defined(__EMSCRIPTEN__) +typedef struct lw_linux_core_key { + int package_id; + int core_id; +} lw_linux_core_key; + +static int read_linux_topology_value(uint32_t cpu, const char* name, int* value) { + char path[160]; + FILE* file; + int written = + snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%u/topology/%s", cpu, name); + if (written <= 0 || (size_t)written >= sizeof(path)) { + return 0; + } + file = fopen(path, "r"); + if (file == NULL) { + return 0; + } + if (fscanf(file, "%d", value) != 1) { + fclose(file); + return 0; + } + return fclose(file) == 0; +} + +static uint32_t linux_cpu_quota(void) { + FILE* file = fopen("/sys/fs/cgroup/cpu.max", "r"); + char quota_text[32]; + unsigned long long quota; + unsigned long long period; + if (file != NULL) { + int fields = fscanf(file, "%31s %llu", quota_text, &period); + fclose(file); + if (fields == 2 && strcmp(quota_text, "max") != 0) { + char* end = NULL; + errno = 0; + quota = strtoull(quota_text, &end, 10); + if (errno == 0 && end != quota_text && *end == '\0' && quota > 0u && period > 0u) { + return clamp_positive_u64((quota + period - 1u) / period); + } + } + } + file = fopen("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r"); + if (file != NULL) { + long long signed_quota = -1; + int fields = fscanf(file, "%lld", &signed_quota); + fclose(file); + if (fields == 1 && signed_quota > 0) { + file = fopen("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r"); + if (file != NULL) { + int period_fields = fscanf(file, "%llu", &period); + fclose(file); + if (period_fields == 1 && period > 0u) { + quota = (unsigned long long)signed_quota; + return clamp_positive_u64((quota + period - 1u) / period); + } + } + } + } + return 0u; +} + +static void detect_linux_topology(lw_cpu_topology* topology) { + cpu_set_t affinity; + lw_linux_core_key keys[CPU_SETSIZE]; + uint32_t logical = 0u; + uint32_t physical = 0u; + uint32_t cpu; + uint32_t quota; + CPU_ZERO(&affinity); + if (sched_getaffinity(0, sizeof(affinity), &affinity) == 0) { + for (cpu = 0u; cpu < (uint32_t)CPU_SETSIZE; ++cpu) { + int package_id; + int core_id; + uint32_t index; + if (!CPU_ISSET((int)cpu, &affinity)) { + continue; + } + ++logical; + if (!read_linux_topology_value(cpu, "physical_package_id", &package_id) || + !read_linux_topology_value(cpu, "core_id", &core_id)) { + continue; + } + for (index = 0u; index < physical; ++index) { + if (keys[index].package_id == package_id && keys[index].core_id == core_id) { + break; + } + } + if (index == physical && physical < (uint32_t)CPU_SETSIZE) { + keys[physical].package_id = package_id; + keys[physical].core_id = core_id; + ++physical; + } + } + } + if (logical == 0u) { + long online = sysconf(_SC_NPROCESSORS_ONLN); + logical = online > 0 ? clamp_positive_u64((uint64_t)online) : 1u; + } + quota = linux_cpu_quota(); + topology->logical_processors = logical; + topology->available_processors = quota != 0u && quota < logical ? quota : logical; + topology->physical_cores = physical == 0u ? topology->available_processors : physical; +} +#endif + +void lw_cpu_topology_detect(lw_cpu_topology* topology) { + uint32_t logical = 1u; + uint32_t physical = 1u; + uint32_t available = 1u; + if (topology == NULL) { + return; + } + memset(topology, 0, sizeof(*topology)); +#if defined(__EMSCRIPTEN__) + /* The offline module is intentionally built without pthreads. */ +#elif defined(_WIN32) + { + SYSTEM_INFO system_info; + DWORD_PTR process_mask = 0u; + DWORD_PTR system_mask = 0u; + GetSystemInfo(&system_info); + logical = system_info.dwNumberOfProcessors == 0u + ? 1u + : (uint32_t)system_info.dwNumberOfProcessors; + available = logical; + if (GetProcessAffinityMask(GetCurrentProcess(), &process_mask, &system_mask)) { + uint32_t affinity_count = count_affinity_bits(process_mask); + if (affinity_count != 0u && affinity_count < available) { + available = affinity_count; + } + } + physical = windows_physical_core_count(); + if (physical == 0u) { + physical = available; + } + if (physical > available) { + physical = available; + } + } +#elif defined(__linux__) + detect_linux_topology(topology); + logical = topology->logical_processors; + physical = topology->physical_cores; + available = topology->available_processors; +#elif defined(__APPLE__) + { + size_t size = sizeof(logical); + if (sysctlbyname("hw.logicalcpu", &logical, &size, NULL, 0u) != 0 || logical == 0u) { + logical = 1u; + } + size = sizeof(physical); + if (sysctlbyname("hw.physicalcpu", &physical, &size, NULL, 0u) != 0 || physical == 0u) { + physical = logical; + } + available = logical; + } +#else + { + long online = sysconf(_SC_NPROCESSORS_ONLN); + logical = online > 0 ? clamp_positive_u64((uint64_t)online) : 1u; + physical = logical; + available = logical; + } +#endif + topology->logical_processors = logical == 0u ? 1u : logical; + topology->available_processors = available == 0u ? 1u : available; + topology->physical_cores = physical == 0u ? topology->available_processors : physical; + if (topology->available_processors > topology->logical_processors) { + topology->available_processors = topology->logical_processors; + } + if (topology->physical_cores > topology->logical_processors) { + topology->physical_cores = topology->logical_processors; + } + topology->smt_width = + (topology->logical_processors + topology->physical_cores - 1u) / topology->physical_cores; + if (topology->smt_width == 0u) { + topology->smt_width = 1u; + } +} + +#if INTPTR_MAX > INT32_MAX && !defined(__EMSCRIPTEN__) +static uint32_t cap_native_count(uint32_t count) { + if (count == 0u) { + count = LW_DEFAULT_NATIVE_PROCESSOR_FALLBACK; + } + return count > LW_DEFAULT_NATIVE_PARALLEL_CAP ? LW_DEFAULT_NATIVE_PARALLEL_CAP : count; +} +#endif + +uint32_t lw_parallel_default_line_worker_count(const lw_cpu_topology* topology) { +#if INTPTR_MAX <= INT32_MAX || defined(__EMSCRIPTEN__) + (void)topology; + return 1u; +#else + return cap_native_count(topology == NULL ? 0u : topology->available_processors); +#endif +} + +uint32_t lw_parallel_default_det_thread_count(const lw_cpu_topology* topology) { +#if INTPTR_MAX <= INT32_MAX || defined(__EMSCRIPTEN__) + (void)topology; + return 1u; +#else + uint32_t count = 0u; + if (topology != NULL) { + count = topology->physical_cores; + if (topology->available_processors != 0u && topology->available_processors < count) { + count = topology->available_processors; + } + } + return cap_native_count(count); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/cpu_topology.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/cpu_topology.h new file mode 100644 index 00000000..896fff17 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/cpu_topology.h @@ -0,0 +1,21 @@ +#ifndef LW_CPU_TOPOLOGY_H +#define LW_CPU_TOPOLOGY_H + +/* Best-effort CPU availability used by the private parallel policy. The + * values describe processors available to this process, not necessarily the + * complete host, so affinity and common Linux CPU quotas can reduce them. */ + +#include + +typedef struct lw_cpu_topology { + uint32_t logical_processors; + uint32_t physical_cores; + uint32_t available_processors; + uint32_t smt_width; +} lw_cpu_topology; + +void lw_cpu_topology_detect(lw_cpu_topology* topology); +uint32_t lw_parallel_default_line_worker_count(const lw_cpu_topology* topology); +uint32_t lw_parallel_default_det_thread_count(const lw_cpu_topology* topology); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_head_parallel_internal.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_head_parallel_internal.c new file mode 100644 index 00000000..dc404ca6 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_head_parallel_internal.c @@ -0,0 +1,142 @@ +#include "ctc_head_parallel_internal.h" + +#include "../simd/simd_kernels.h" + +#include + +/* Minimum total MACs before worker fan-out pays for synchronization. */ +#define LW_CTC_HEAD_PARALLEL_MIN_WORK (4ull * 1000ull * 1000ull) + +uint32_t lw_ctc_head_parallel_worker_count(const lw_thread_pool* pool, + uint32_t max_workers, uint32_t rows, + uint32_t inner_dimension, uint32_t columns) { + uint32_t pool_workers; + uint64_t work; + if (pool == NULL || max_workers <= 1u || rows < 8u) { + return 1u; + } + work = (uint64_t)rows * inner_dimension * columns; + if (work < LW_CTC_HEAD_PARALLEL_MIN_WORK) { + return 1u; + } + pool_workers = lw_thread_pool_worker_count(pool); + if (max_workers > pool_workers) { + max_workers = pool_workers; + } + /* Keep at least one 4-row block per worker. */ + if (max_workers > rows / 4u) { + max_workers = rows / 4u; + } + return max_workers <= 1u ? 1u : max_workers; +} + +/* Row ranges are split in units of 4-row blocks: the packed kernels process + * rows in blocks of 4 and silently drop a tail that is not a multiple of 4, + * so slice boundaries must stay block-aligned. */ +static uint32_t row_range_begin(uint32_t rows, uint32_t worker_count, uint32_t worker_index) { + uint32_t blocks = rows / 4u; + uint32_t base = blocks / worker_count; + uint32_t remainder = blocks % worker_count; + return (worker_index * base + (worker_index < remainder ? worker_index : remainder)) * 4u; +} + +typedef struct lw_ctc_head_argmax_context { + lw_rec_ctc_argmax_scores_fn kernel; + const float* input; + const float* packed_weights; + const float* bias; + uint32_t* best_indices; + float* scores; + uint32_t rows; + uint32_t inner_dimension; + uint32_t columns; +} lw_ctc_head_argmax_context; + +static void execute_argmax_scores_rows(void* context_void, uint32_t worker_index, + uint32_t worker_count) { + lw_ctc_head_argmax_context* context = (lw_ctc_head_argmax_context*)context_void; + uint32_t row_begin = row_range_begin(context->rows, worker_count, worker_index); + uint32_t row_end = row_range_begin(context->rows, worker_count, worker_index + 1u); + if (row_end <= row_begin) { + return; + } + /* Rows are independent and each output element accumulates in the serial + * kernel's order, so the row split is bit-identical. */ + context->kernel( + context->input + (size_t)row_begin * context->inner_dimension, + context->packed_weights, context->bias, context->best_indices + row_begin, + context->scores + row_begin, row_end - row_begin, context->inner_dimension, + context->columns); +} + +void lw_ctc_head_argmax_scores_parallel_f32(lw_thread_pool* pool, uint32_t worker_count, + lw_rec_ctc_argmax_scores_fn kernel, + const float* input, const float* packed_weights, + const float* bias, uint32_t* best_indices, + float* scores, uint32_t rows, + uint32_t inner_dimension, uint32_t columns) { + lw_ctc_head_argmax_context context; + if (pool == NULL || worker_count <= 1u) { + kernel(input, packed_weights, bias, best_indices, scores, rows, + inner_dimension, columns); + return; + } + context.kernel = kernel; + context.input = input; + context.packed_weights = packed_weights; + context.bias = bias; + context.best_indices = best_indices; + context.scores = scores; + context.rows = rows; + context.inner_dimension = inner_dimension; + context.columns = columns; + lw_thread_pool_run(pool, worker_count, execute_argmax_scores_rows, &context); +} + +typedef struct lw_ctc_head_bias_argmax_context { + const float* input; + const float* packed_weights; + const float* bias; + float* output; + uint32_t* best_indices; + uint32_t rows; + uint32_t inner_dimension; + uint32_t columns; +} lw_ctc_head_bias_argmax_context; + +static void execute_bias_argmax_rows(void* context_void, uint32_t worker_index, + uint32_t worker_count) { + lw_ctc_head_bias_argmax_context* context = (lw_ctc_head_bias_argmax_context*)context_void; + uint32_t row_begin = row_range_begin(context->rows, worker_count, worker_index); + uint32_t row_end = row_range_begin(context->rows, worker_count, worker_index + 1u); + if (row_end <= row_begin) { + return; + } + lw_avx2_packed_matmul_bias_argmax_f32( + context->input + (size_t)row_begin * context->inner_dimension, context->packed_weights, + context->bias, context->output + (size_t)row_begin * context->columns, + context->best_indices + row_begin, 1u, row_end - row_begin, context->inner_dimension, + context->columns); +} + +void lw_ctc_head_bias_argmax_parallel_f32(lw_thread_pool* pool, uint32_t worker_count, + const float* input, const float* packed_weights, + const float* bias, float* output, + uint32_t* best_indices, uint32_t rows, + uint32_t inner_dimension, uint32_t columns) { + lw_ctc_head_bias_argmax_context context; + if (pool == NULL || worker_count <= 1u) { + lw_avx2_packed_matmul_bias_argmax_f32(input, packed_weights, bias, output, best_indices, + 1u, rows, inner_dimension, columns); + return; + } + context.input = input; + context.packed_weights = packed_weights; + context.bias = bias; + context.output = output; + context.best_indices = best_indices; + context.rows = rows; + context.inner_dimension = inner_dimension; + context.columns = columns; + lw_thread_pool_run(pool, worker_count, execute_bias_argmax_rows, &context); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_head_parallel_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_head_parallel_internal.h new file mode 100644 index 00000000..d8e038f1 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_head_parallel_internal.h @@ -0,0 +1,34 @@ +#ifndef LW_CTC_HEAD_PARALLEL_INTERNAL_H +#define LW_CTC_HEAD_PARALLEL_INTERNAL_H + +/* Row-parallel wrappers for the CTC head packed-matmul kernels. Each output + * element is accumulated in exactly the same order as in the serial kernel + * (rows are independent), so results are bit-identical; only the row range + * is split across pool workers. */ + +#include "parallel_internal.h" +#include "rec_backend_kernels_internal.h" + +#include + +/* Effective worker count for a CTC head projection: 1 (serial) when the pool + * is missing, the cap is 1, the row count is tiny, or the total MAC count is + * too small to amortize synchronization. */ +uint32_t lw_ctc_head_parallel_worker_count(const lw_thread_pool* pool, + uint32_t max_workers, uint32_t rows, + uint32_t inner_dimension, uint32_t columns); + +void lw_ctc_head_argmax_scores_parallel_f32(lw_thread_pool* pool, uint32_t worker_count, + lw_rec_ctc_argmax_scores_fn kernel, + const float* input, const float* packed_weights, + const float* bias, uint32_t* best_indices, + float* scores, uint32_t rows, + uint32_t inner_dimension, uint32_t columns); + +void lw_ctc_head_bias_argmax_parallel_f32(lw_thread_pool* pool, uint32_t worker_count, + const float* input, const float* packed_weights, + const float* bias, float* output, + uint32_t* best_indices, uint32_t rows, + uint32_t inner_dimension, uint32_t columns); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_projection_internal.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_projection_internal.c new file mode 100644 index 00000000..d0cb98cd --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_projection_internal.c @@ -0,0 +1,105 @@ +#include "ctc_projection_internal.h" +#include "ctc_head_parallel_internal.h" +#include "packed_matmul_internal.h" +#include "rec_backend_kernels_internal.h" +#include "../simd/simd_kernels.h" +#include "lwm_read.h" + +#include +#include +#if defined(__EMSCRIPTEN__) +#include +#endif + +static const float* constant_f32(const lw_model* model, uint32_t index) { + const uint8_t* tensor = model->bytes + (size_t)model->tensor_offset + + (size_t)index * LWM_V0_TENSOR_SIZE; + return (const float*)(const void*)(model->bytes + (size_t)lwm_read_u64(tensor + 48u)); +} + +lw_status lw_x64_rec_ctc_prepare_shared(const lw_model* model, const lw_session* session, + const lw_x64_rec_ctc_tail* source, + lw_x64_rec_ctc_tail* tail, lw_error* error) { + uint64_t count; + (void)session; + if (model == NULL || tail == NULL || !tail->enabled || + !lw_packed_matmul_weight_count(tail->inner, tail->classes, &count) || + count > SIZE_MAX / sizeof(float)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid CTC projection descriptor"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (source != NULL && source->enabled != 0u && source->packed_weights != NULL && + source->weight_tensor == tail->weight_tensor && source->inner == tail->inner && + source->classes == tail->classes) { + tail->packed_weights = source->packed_weights; + tail->packed_weights_borrowed = 1u; + tail->bias = constant_f32(model, tail->bias_tensor); + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + } + tail->packed_weights = (float*)malloc((size_t)count * sizeof(float)); + if (tail->packed_weights == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "CTC packed weight allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + lw_pack_matmul_weights_f32(constant_f32(model, tail->weight_tensor), tail->inner, + tail->classes, tail->packed_weights); + tail->bias = constant_f32(model, tail->bias_tensor); + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +lw_status lw_x64_rec_ctc_prepare(const lw_model* model, const lw_session* session, + lw_x64_rec_ctc_tail* tail, lw_error* error) { + return lw_x64_rec_ctc_prepare_shared(model, session, NULL, tail, error); +} + +void lw_x64_rec_ctc_free(lw_x64_rec_ctc_tail* tail) { + if (tail == NULL) return; + if (tail->packed_weights_borrowed == 0u) free(tail->packed_weights); + tail->packed_weights = NULL; + tail->packed_weights_borrowed = 0u; + tail->bias = NULL; +} + +lw_status lw_x64_rec_ctc_execute(const lw_x64_rec_program* program, + lw_x64_rec_instance* instance, const float* activation, + lw_error* error) { + if (program == NULL || instance == NULL || activation == NULL || !program->ctc.enabled || + instance->ctc_scores == NULL || instance->best_indices == NULL || + instance->best_probabilities == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid CTC runtime state"); + return LW_STATUS_INVALID_ARGUMENT; + } + memset(instance->best_probabilities, 0, (size_t)program->ctc.rows * sizeof(*instance->best_probabilities)); + /* The logit tensor is never materialized: the argmax pass keeps only the + * per-row maximum and index, and the probability pass recomputes one row + * of logits per emitted step. Rows are independent, so the argmax pass + * may fan out over the borrowed intra-op pool bit-identically. */ + { + const lw_rec_backend_kernels* kernels = lw_rec_backend_kernels_current(); + uint32_t head_workers = lw_ctc_head_parallel_worker_count( + instance->thread_pool, instance->intra_op_workers, program->ctc.rows, + program->ctc.inner, program->ctc.classes); + lw_ctc_head_argmax_scores_parallel_f32( + instance->thread_pool, head_workers, kernels->ctc_argmax_scores, + activation, program->ctc.packed_weights, + program->ctc.bias, instance->best_indices, instance->ctc_scores, program->ctc.rows, + program->ctc.inner, program->ctc.classes); + kernels->ctc_probabilities( + activation, program->ctc.packed_weights, program->ctc.bias, + instance->best_indices, instance->ctc_scores, instance->best_probabilities, + program->ctc.rows, program->ctc.inner, program->ctc.classes); +#if defined(__EMSCRIPTEN__) + { + static int reported = 0; + if (!reported) { + fprintf(stderr, "LW_WASM_COMPILED_REC ctc=simd128\n"); + reported = 1; + } + } +#endif + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_projection_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_projection_internal.h new file mode 100644 index 00000000..07e0a697 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/ctc_projection_internal.h @@ -0,0 +1,16 @@ +#ifndef LW_X64_REC_CTC_PROJECTION_INTERNAL_H +#define LW_X64_REC_CTC_PROJECTION_INTERNAL_H + +#include "x64_rec_backend_internal.h" + +lw_status lw_x64_rec_ctc_prepare(const lw_model* model, const lw_session* session, + lw_x64_rec_ctc_tail* tail, lw_error* error); +lw_status lw_x64_rec_ctc_prepare_shared(const lw_model* model, const lw_session* session, + const lw_x64_rec_ctc_tail* source, + lw_x64_rec_ctc_tail* tail, lw_error* error); +void lw_x64_rec_ctc_free(lw_x64_rec_ctc_tail* tail); +lw_status lw_x64_rec_ctc_execute(const lw_x64_rec_program* program, + lw_x64_rec_instance* instance, const float* activation, + lw_error* error); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/error_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/error_internal.h new file mode 100644 index 00000000..57af6510 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/error_internal.h @@ -0,0 +1,8 @@ +#ifndef LW_ERROR_INTERNAL_H +#define LW_ERROR_INTERNAL_H + +#include "lw_infer.h" + +void lw_set_error(lw_error* error, lw_status status, const char* message); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/execution_plan.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/execution_plan.c new file mode 100644 index 00000000..b2250617 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/execution_plan.c @@ -0,0 +1,334 @@ +#include "session_internal.h" +#include "operator_internal.h" + +#include "lwm_read.h" + +#include +#include + + +static int constant_scalar_f32(const lw_session* session, uint32_t tensor_index, float expected) { + const lw_runtime_tensor* tensor = &session->tensors[tensor_index]; + const uint8_t* disk; + uint64_t data_offset; + float value; + if (tensor->dtype != LW_DTYPE_F32 || tensor->byte_size != sizeof(float) || + (tensor->flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u) { + return 0; + } + disk = session->model->bytes + (size_t)session->model->tensor_offset + + (size_t)tensor_index * LWM_V0_TENSOR_SIZE; + data_offset = lwm_read_u64(disk + 48u); + memcpy(&value, session->model->bytes + (size_t)data_offset, sizeof(value)); + return value == expected; +} + +int lw_match_fused_gelu(const lw_session* session, uint32_t node_index, + lw_fused_gelu_match* match) { + const lw_model* model; + uint32_t index; + const lw_runtime_tensor* source; + const lw_runtime_tensor* output; + if (session == NULL || match == NULL || session->model == NULL) { + return 0; + } + model = session->model; + if (node_index > model->info.node_count || model->info.node_count - node_index < 5u) { + return 0; + } + for (index = 0u; index < 5u; ++index) { + const uint8_t* node = model->bytes + (size_t)model->node_offset + + (size_t)(node_index + index) * LWM_V0_NODE_SIZE; + if (lwm_read_u16(node + 2u) != (index == 1u ? 1u : 2u) || + lwm_read_u16(node + 4u) != 1u) { + return 0; + } + match->inputs[index][0] = lwm_read_u32(node + 8u); + match->inputs[index][1] = index == 1u ? UINT32_MAX : lwm_read_u32(node + 12u); + match->outputs[index] = lwm_read_u32(node + 40u); + } + { + const uint8_t* node0 = model->bytes + (size_t)model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + const uint8_t* node1 = node0 + LWM_V0_NODE_SIZE; + const uint8_t* node2 = node1 + LWM_V0_NODE_SIZE; + const uint8_t* node3 = node2 + LWM_V0_NODE_SIZE; + const uint8_t* node4 = node3 + LWM_V0_NODE_SIZE; + if (lwm_read_u16(node0) != LW_OP_DIV || lwm_read_u16(node1) != LW_OP_ERF || + lwm_read_u16(node2) != LW_OP_ADD || lwm_read_u16(node3) != LW_OP_MUL || + lwm_read_u16(node4) != LW_OP_MUL || + match->inputs[1][0] != match->outputs[0] || + (match->inputs[2][0] != match->outputs[1] && + match->inputs[2][1] != match->outputs[1]) || + (match->inputs[3][0] != match->inputs[0][0] && + match->inputs[3][1] != match->inputs[0][0]) || + (match->inputs[3][0] != match->outputs[2] && + match->inputs[3][1] != match->outputs[2]) || + (match->inputs[4][0] != match->outputs[3] && + match->inputs[4][1] != match->outputs[3])) { + return 0; + } + } + if (!constant_scalar_f32(session, match->inputs[0][1], 1.4142135381698608f) || + !constant_scalar_f32(session, + match->inputs[2][0] == match->outputs[1] ? match->inputs[2][1] + : match->inputs[2][0], + 1.0f) || + !constant_scalar_f32(session, + match->inputs[4][0] == match->outputs[3] ? match->inputs[4][1] + : match->inputs[4][0], + 0.5f)) { + return 0; + } + source = &session->tensors[match->inputs[0][0]]; + output = &session->tensors[match->outputs[4]]; + if (source->dtype != LW_DTYPE_F32 || output->dtype != LW_DTYPE_F32 || + source->byte_size == 0u || source->byte_size != output->byte_size) { + return 0; + } + for (index = 0u; index < 4u; ++index) { + const lw_runtime_tensor* temporary = &session->tensors[match->outputs[index]]; + if (temporary->dtype != LW_DTYPE_F32 || temporary->byte_size != source->byte_size) { + return 0; + } + } + return 1; +} + +#if defined(LW_EXPERIMENTAL_FUSION_MEMORY) +static int32_t tensor_last_use_node(const lw_session* session, uint32_t tensor_index) { + const lw_model* model = session->model; + int32_t last_use = -1; + uint32_t node_index; + for (node_index = 0u; node_index < model->info.node_count; ++node_index) { + const uint8_t* node = model->bytes + (size_t)model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + uint16_t input_count = lwm_read_u16(node + 2u); + uint32_t input_index; + for (input_index = 0u; input_index < input_count; ++input_index) { + if (lwm_read_u32(node + 8u + (size_t)input_index * sizeof(uint32_t)) == tensor_index) { + last_use = (int32_t)node_index; + } + } + } + for (node_index = 0u; node_index < model->info.output_count; ++node_index) { + if (lwm_read_u32(model->bytes + (size_t)model->output_offset + + (size_t)node_index * sizeof(uint32_t)) == tensor_index) { + last_use = (int32_t)model->info.node_count; + } + } + return last_use; +} +#endif + +#if defined(LW_EXPERIMENTAL_CTC_TILED) +static int match_ctc_greedy_tail(const lw_session* session, uint32_t* skip_tensor) { + const lw_model* model; + const uint8_t* node; + const uint8_t* params; + const lw_runtime_tensor* logits; + const lw_runtime_tensor* output; + uint32_t graph_output; + uint32_t input_index; + uint32_t output_index; + int32_t axis; + uint32_t index; + if (session == NULL || skip_tensor == NULL || session->model == NULL || + session->ctc_greedy_plan_enabled == 0u) { + return 0; + } + model = session->model; + if (model->info.input_count != 1u || model->info.output_count != 1u || + model->info.node_count == 0u) { + return 0; + } + node = model->bytes + (size_t)model->node_offset + + (size_t)(model->info.node_count - 1u) * LWM_V0_NODE_SIZE; + if (lwm_read_u16(node) != LW_OP_SOFTMAX || lwm_read_u16(node + 2u) != 1u || + lwm_read_u16(node + 4u) != 1u) { + return 0; + } + input_index = lwm_read_u32(node + 8u); + output_index = lwm_read_u32(node + 40u); + graph_output = lwm_read_u32(model->bytes + (size_t)model->output_offset); + if (output_index != graph_output || input_index >= model->info.tensor_count || + output_index >= model->info.tensor_count) { + return 0; + } + logits = &session->tensors[input_index]; + output = &session->tensors[output_index]; + if (logits->dtype != LW_DTYPE_F32 || output->dtype != LW_DTYPE_F32 || logits->rank < 2u || + logits->rank != output->rank || output->byte_size == 0u) { + return 0; + } + for (index = 0u; index < logits->rank; ++index) { + if (logits->dimensions[index] != output->dimensions[index]) { + return 0; + } + } + params = model->bytes + (size_t)lwm_read_u64(node + 56u); + axis = lwm_read_i32(params + 4u); + if (axis < 0) { + axis += (int32_t)logits->rank; + } + if (axis != (int32_t)logits->rank - 1) { + return 0; + } + *skip_tensor = output_index; + return 1; +} + +static lw_status prepare_ctc_logits_scratch(lw_session* session, lw_error* error) { + const lw_runtime_tensor* output; + uint32_t graph_output; + uint32_t class_count; + uint32_t time_steps; + uint32_t rows; + uint64_t elements; + if (session->ctc_greedy_skip_tensor == UINT32_MAX || + !lw_simd_level_is_avx2(session->cpu.simd)) { + return LW_STATUS_OK; + } + graph_output = session->ctc_greedy_skip_tensor; + output = &session->tensors[graph_output]; + /* The tiled kernel currently targets the packed REC shape [1,T,V]. + * Other terminal Softmax shapes retain the safe output-elision plan but + * do not reserve a scratch buffer that the executor cannot consume. */ + if (output->rank != 3u || output->dimensions[0] != 1 || + output->dimensions[output->rank - 1u] <= 0 || + output->dimensions[output->rank - 2u] <= 0) { + return LW_STATUS_OK; + } + class_count = (uint32_t)output->dimensions[output->rank - 1u]; + time_steps = (uint32_t)output->dimensions[output->rank - 2u]; + rows = time_steps < 8u ? time_steps : 8u; + elements = (uint64_t)rows * class_count; + if (rows == 0u || elements > (uint64_t)(SIZE_MAX / sizeof(float))) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "CTC greedy scratch size overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + session->ctc_logits_scratch = (float*)malloc((size_t)elements * sizeof(float)); + if (session->ctc_logits_scratch == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate CTC greedy scratch"); + return LW_STATUS_OUT_OF_MEMORY; + } + session->ctc_logits_scratch_rows = rows; + session->ctc_logits_scratch_elements = elements; + return LW_STATUS_OK; +} +#endif + +void lw_free_execution_plan(lw_session* session) { + if (session == NULL) { + return; + } + free(session->fused_gelu_start_nodes); + free(session->fused_gelu_skip_tensors); + free(session->ctc_logits_scratch); + session->fused_gelu_start_nodes = NULL; + session->fused_gelu_skip_tensors = NULL; + session->ctc_greedy_skip_tensor = UINT32_MAX; + session->ctc_logits_scratch = NULL; + session->ctc_logits_scratch_rows = 0u; + session->ctc_logits_scratch_elements = 0u; +} + +lw_status lw_prepare_execution_plan(lw_session* session, lw_error* error) { + uint32_t node_index; + lw_free_execution_plan(session); +#if defined(LW_EXPERIMENTAL_CTC_TILED) + (void)match_ctc_greedy_tail(session, &session->ctc_greedy_skip_tensor); + { + lw_status scratch_status = prepare_ctc_logits_scratch(session, error); + if (scratch_status != LW_STATUS_OK) { + lw_free_execution_plan(session); + return scratch_status; + } + } +#endif +#if defined(LW_EXPERIMENTAL_FUSION_MEMORY) + if (!lw_simd_level_is_avx2(session->cpu.simd)) { + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + } + if (session->model->info.node_count != 0u) { + session->fused_gelu_start_nodes = (uint8_t*)calloc( + session->model->info.node_count, sizeof(*session->fused_gelu_start_nodes)); + } + if (session->model->info.tensor_count != 0u) { + session->fused_gelu_skip_tensors = (uint8_t*)calloc( + session->model->info.tensor_count, sizeof(*session->fused_gelu_skip_tensors)); + } + if ((session->model->info.node_count != 0u && session->fused_gelu_start_nodes == NULL) || + (session->model->info.tensor_count != 0u && session->fused_gelu_skip_tensors == NULL)) { + lw_free_execution_plan(session); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate fused execution plan"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (node_index = 0u; node_index < session->model->info.node_count; ++node_index) { + lw_fused_gelu_match match; + uint32_t output_index; + if (!lw_match_fused_gelu(session, node_index, &match)) { + continue; + } + for (output_index = 0u; output_index < 4u; ++output_index) { + if (tensor_last_use_node(session, match.outputs[output_index]) != + (int32_t)(node_index + output_index + 1u)) { + break; + } + } + if (output_index != 4u) { + continue; + } + session->fused_gelu_start_nodes[node_index] = 1u; + session->fused_gelu_skip_tensors[match.outputs[0]] = 1u; + session->fused_gelu_skip_tensors[match.outputs[1]] = 1u; + session->fused_gelu_skip_tensors[match.outputs[2]] = 1u; + session->fused_gelu_skip_tensors[match.outputs[3]] = 1u; + node_index += 4u; + } +#else + (void)node_index; +#endif + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +int lw_execution_plan_is_fused_gelu_start(const lw_session* session, uint32_t node_index) { +#if defined(LW_EXPERIMENTAL_FUSION_MEMORY) + return session != NULL && session->fused_gelu_start_nodes != NULL && + node_index < session->model->info.node_count && + session->fused_gelu_start_nodes[node_index] != 0u; +#else + (void)session; + (void)node_index; + return 0; +#endif +} + +int lw_execution_plan_is_fused_gelu_skip(const lw_session* session, uint32_t tensor_index) { +#if defined(LW_EXPERIMENTAL_FUSION_MEMORY) + return session != NULL && session->fused_gelu_skip_tensors != NULL && + tensor_index < session->model->info.tensor_count && + session->fused_gelu_skip_tensors[tensor_index] != 0u; +#else + (void)session; + (void)tensor_index; + return 0; +#endif +} + +int lw_execution_plan_is_ctc_greedy_skip(const lw_session* session, uint32_t tensor_index) { +#if defined(LW_EXPERIMENTAL_CTC_TILED) + return session != NULL && session->ctc_greedy_skip_tensor == tensor_index; +#else + (void)session; + (void)tensor_index; + return 0; +#endif +} + +int lw_execution_plan_is_skipped_tensor(const lw_session* session, uint32_t tensor_index) { + return lw_execution_plan_is_fused_gelu_skip(session, tensor_index) || + lw_execution_plan_is_ctc_greedy_skip(session, tensor_index); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/executor.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/executor.c new file mode 100644 index 00000000..03194cd4 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/executor.c @@ -0,0 +1,1780 @@ +#include "executor_internal.h" + +/* + * Small graph interpreter for the operator subset emitted by the converter. + * Constants point into validated model bytes; intermediate tensors point into + * the preplanned workspace. Dispatch must stay allocation-free during a run. + */ + +#include "lwm_read.h" +#include "ctc_head_parallel_internal.h" +#include "packed_conv_internal.h" +#include "packed_conv3x3_internal.h" +#include "parallel_internal.h" +#include "scalar_kernels.h" +#include "session_internal.h" +#include "cpu_features.h" +#include "simd_kernels.h" +#include "operator_internal.h" + +#include +#include + +#define LW_PARALLEL_CONV_MIN_MULTIPLY_ADDS UINT64_C(8000000) +#define LW_PARALLEL_CONV_TRANSPOSE_MIN_MULTIPLY_ADDS UINT64_C(8000000) + + + +static float read_f32(const uint8_t* bytes) { + uint32_t bits = lwm_read_u32(bytes); + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static uint64_t tensor_element_count(const lw_runtime_tensor* tensor) { + return tensor->byte_size / sizeof(float); +} +static void profile_layout_candidate_nodes(lw_execution_profile* profile, + const lw_session* session, uint32_t node_limit) { + const lw_model* model; + uint32_t node_index; + if (profile == NULL || session == NULL || session->model == NULL || + profile->layout_analysis_runs != 0u) { + return; + } + model = session->model; + if (node_limit > model->info.node_count) { + node_limit = model->info.node_count; + } + for (node_index = 0u; node_index < node_limit; ++node_index) { + const uint8_t* node = + model->bytes + (size_t)model->node_offset + (size_t)node_index * LWM_V0_NODE_SIZE; + uint16_t operation = lwm_read_u16(node); + uint16_t input_count = lwm_read_u16(node + 2u); + uint32_t input_index; + uint32_t output_index; + const lw_runtime_tensor* input_tensor; + const lw_runtime_tensor* output_tensor; + uint32_t groups; + uint64_t param_offset; + const uint8_t* params; + if (operation != LW_OP_CONV || input_count < 2u) { + continue; + } + input_index = lwm_read_u32(node + 8u); + output_index = lwm_read_u32(node + 40u); + if (input_index >= session->model->info.tensor_count || output_index >= session->model->info.tensor_count) { + continue; + } + input_tensor = &session->tensors[input_index]; + output_tensor = &session->tensors[output_index]; + param_offset = lwm_read_u64(node + 56u); + if (param_offset == 0u || param_offset > model->byte_count - 8u) { + continue; + } + params = model->bytes + (size_t)param_offset; + groups = lwm_read_u32(params + 4u); + /* Conservative eligibility: batch-one rank-4, group-one Conv. The + * current executor still selects NCHW, so every candidate is a + * measured fallback until a verified NHWC kernel is added. */ + if (input_tensor->rank != 4u || output_tensor->rank != 4u || + input_tensor->dimensions[0] != 1 || output_tensor->dimensions[0] != 1 || + groups != 1u) { + continue; + } + if (profile->layout_candidate_nodes != UINT64_MAX) { + ++profile->layout_candidate_nodes; + } + if (profile->layout_fallback_nodes != UINT64_MAX) { + ++profile->layout_fallback_nodes; + } + } + profile->layout_analysis_runs = 1u; +} + +static void profile_add_layout_value(uint64_t* destination, uint64_t value) { + if (destination != NULL && *destination <= UINT64_MAX - value) { + *destination += value; + } else if (destination != NULL) { + *destination = UINT64_MAX; + } +} +static const float* tensor_input_data(const lw_session* session, uint32_t tensor_index, + uint32_t graph_input_index, const float* graph_input) { + const lw_runtime_tensor* tensor = &session->tensors[tensor_index]; + const lw_model* model = session->model; + if (tensor_index == graph_input_index) { + return graph_input; + } + if ((tensor->flags & LWM_V0_TENSOR_FLAG_CONSTANT) != 0u) { + /* Constant payloads are zero-copy views into immutable model bytes. */ + const uint8_t* disk = + model->bytes + (size_t)model->tensor_offset + (size_t)tensor_index * LWM_V0_TENSOR_SIZE; + uint64_t data_offset = lwm_read_u64(disk + 48); + return (const float*)(const void*)(model->bytes + (size_t)data_offset); + } + if (tensor->workspace_offset == UINT64_MAX) { + return NULL; + } + return (const float*)(const void*)(session->workspace + (size_t)tensor->workspace_offset); +} + +static float* tensor_output_data(lw_session* session, uint32_t tensor_index) { + const lw_runtime_tensor* tensor = &session->tensors[tensor_index]; + if (tensor->workspace_offset == UINT64_MAX) { + return NULL; + } + return (float*)(void*)(session->workspace + (size_t)tensor->workspace_offset); +} + +typedef struct lw_parallel_conv_context { + const float* input; + const float* weights; + const float* bias; + float* output; + int32_t input_dimensions[4]; + int32_t weight_dimensions[4]; + int32_t output_dimensions[4]; + int32_t kernel[2]; + int32_t strides[2]; + int32_t dilations[2]; + int32_t pads[4]; + uint32_t groups; + uint32_t packed; + lw_packed_conv1x1_kernel_fn conv1x1_kernel; + lw_packed_conv3x3_kernel_fn conv3x3_kernel; + lw_status statuses[LW_PARALLEL_MAX_WORKERS]; +} lw_parallel_conv_context; + +typedef struct lw_parallel_conv_transpose_context { + const float* input; + const float* weights; + const float* bias; + float* output; + int32_t input_dimensions[4]; + int32_t weight_dimensions[4]; + int32_t output_dimensions[4]; + int32_t kernel[2]; + int32_t strides[2]; + int32_t dilations[2]; + int32_t pads[4]; + uint32_t groups; + uint32_t bias_count; + lw_simd_level simd_level; +} lw_parallel_conv_transpose_context; + +static uint64_t multiply_saturated(uint64_t left, uint64_t right) { + return right != 0u && left > UINT64_MAX / right ? UINT64_MAX : left * right; +} + +static uint32_t parallel_conv_worker_count(const lw_session* session, + const lw_parallel_conv_context* context) { + uint32_t output_channels = (uint32_t)context->output_dimensions[1]; + uint32_t maximum_workers; + uint64_t work; + int depthwise; + if (session->intra_op_thread_count <= 1u || context->input_dimensions[0] != 1 || + context->output_dimensions[0] != 1 || output_channels == 0u) { + return 1u; + } + depthwise = context->groups == (uint32_t)context->input_dimensions[1] && + context->groups == output_channels && context->weight_dimensions[1] == 1; + if (context->groups != 1u && !depthwise) { + return 1u; + } + if (context->packed != 0u) { + const uint32_t output_tile = + context->packed == LW_PREPARED_CONSTANT_CONV3X3_STRIDE2_PACKED8 + ? LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE + : LW_PACKED_CONV1X1_OUTPUT_TILE; + if (output_channels % output_tile != 0u) { + return 1u; + } + maximum_workers = output_channels / output_tile; + } else { + maximum_workers = output_channels; + } + if (maximum_workers > session->intra_op_thread_count) { + maximum_workers = session->intra_op_thread_count; + } + if (maximum_workers < 2u) { + return 1u; + } + work = multiply_saturated(output_channels, (uint32_t)context->output_dimensions[2]); + work = multiply_saturated(work, (uint32_t)context->output_dimensions[3]); + work = multiply_saturated(work, (uint32_t)context->weight_dimensions[1]); + work = multiply_saturated(work, (uint32_t)context->kernel[0]); + work = multiply_saturated(work, (uint32_t)context->kernel[1]); + return work >= LW_PARALLEL_CONV_MIN_MULTIPLY_ADDS ? maximum_workers : 1u; +} + +static void execute_parallel_conv_slice(void* opaque, uint32_t worker_index, + uint32_t worker_count) { + lw_parallel_conv_context* context = (lw_parallel_conv_context*)opaque; + uint32_t output_channels = (uint32_t)context->output_dimensions[1]; + uint32_t item_size = + context->packed == LW_PREPARED_CONSTANT_CONV3X3_STRIDE2_PACKED8 + ? LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE + : (context->packed != 0u ? LW_PACKED_CONV1X1_OUTPUT_TILE : 1u); + uint32_t item_count = output_channels / item_size; + uint32_t channel_begin = + (uint32_t)(((uint64_t)item_count * worker_index) / worker_count) * item_size; + uint32_t channel_end = + (uint32_t)(((uint64_t)item_count * (worker_index + 1u)) / worker_count) * item_size; + uint32_t channel_count = channel_end - channel_begin; + uint64_t input_plane = + (uint64_t)(uint32_t)context->input_dimensions[2] * (uint32_t)context->input_dimensions[3]; + uint64_t output_plane = + (uint64_t)(uint32_t)context->output_dimensions[2] * (uint32_t)context->output_dimensions[3]; + uint64_t kernel_plane = (uint64_t)(uint32_t)context->kernel[0] * (uint32_t)context->kernel[1]; + int32_t input_dimensions[4]; + int32_t weight_dimensions[4]; + int32_t output_dimensions[4]; + const float* input = context->input; + const float* weights; + const float* bias = context->bias == NULL ? NULL : context->bias + channel_begin; + float* output = context->output + (size_t)((uint64_t)channel_begin * output_plane); + uint32_t groups = context->groups; + memcpy(input_dimensions, context->input_dimensions, sizeof(input_dimensions)); + memcpy(weight_dimensions, context->weight_dimensions, sizeof(weight_dimensions)); + memcpy(output_dimensions, context->output_dimensions, sizeof(output_dimensions)); + output_dimensions[1] = (int32_t)channel_count; + weight_dimensions[0] = (int32_t)channel_count; + if (context->packed == LW_PREPARED_CONSTANT_CONV1X1_PACKED4) { + if (context->conv1x1_kernel != NULL) { + uint32_t input_channels = (uint32_t)input_dimensions[1]; + weights = context->weights + (size_t)((uint64_t)channel_begin * input_channels); + context->conv1x1_kernel(input, weights, bias, output, input_dimensions, output_dimensions); + context->statuses[worker_index] = LW_STATUS_OK; + return; + } + uint32_t input_channels = (uint32_t)input_dimensions[1]; + weights = context->weights + (size_t)((uint64_t)channel_begin * input_channels); + lw_packed_conv1x1_f32(input, weights, bias, output, input_dimensions, output_dimensions); + context->statuses[worker_index] = LW_STATUS_OK; + return; + } + if (context->packed == LW_PREPARED_CONSTANT_CONV3X3_STRIDE2_PACKED8) { + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + weights = context->weights + + (size_t)((uint64_t)channel_begin * input_channels * 9u); + if (context->conv3x3_kernel != NULL) { + context->conv3x3_kernel(input, weights, bias, output, input_dimensions, + output_dimensions); + context->statuses[worker_index] = LW_STATUS_OK; + return; + } + lw_packed_conv3x3_stride2_pad1_f32( + input, weights, bias, output, input_dimensions, output_dimensions); + context->statuses[worker_index] = LW_STATUS_OK; + return; + } + if (groups == 1u) { + weights = context->weights + + (size_t)((uint64_t)channel_begin * (uint32_t)weight_dimensions[1] * kernel_plane); + } else { + input += (size_t)((uint64_t)channel_begin * input_plane); + weights = context->weights + (size_t)((uint64_t)channel_begin * kernel_plane); + input_dimensions[1] = (int32_t)channel_count; + groups = channel_count; + } + context->statuses[worker_index] = lw_scalar_conv2d_f32( + input, weights, bias, bias == NULL ? 0u : channel_count, output, input_dimensions, + weight_dimensions, output_dimensions, context->kernel, context->strides, context->dilations, + context->pads, groups); +} + +static lw_status dispatch_parallel_conv(lw_session* session, const float* input, + const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t weight_dimensions[4], + const int32_t output_dimensions[4], const int32_t kernel[2], + const int32_t strides[2], const int32_t dilations[2], + const int32_t pads[4], uint32_t groups, uint32_t packed, + lw_packed_conv1x1_kernel_fn conv1x1_kernel, + lw_packed_conv3x3_kernel_fn conv3x3_kernel, + lw_execution_profile* profile) { + lw_parallel_conv_context context; + uint32_t worker_count; + uint32_t worker_index; + memset(&context, 0, sizeof(context)); + context.input = input; + context.weights = weights; + context.bias = bias; + context.output = output; + memcpy(context.input_dimensions, input_dimensions, sizeof(context.input_dimensions)); + memcpy(context.weight_dimensions, weight_dimensions, sizeof(context.weight_dimensions)); + memcpy(context.output_dimensions, output_dimensions, sizeof(context.output_dimensions)); + memcpy(context.kernel, kernel, sizeof(context.kernel)); + memcpy(context.strides, strides, sizeof(context.strides)); + memcpy(context.dilations, dilations, sizeof(context.dilations)); + memcpy(context.pads, pads, sizeof(context.pads)); + context.groups = groups; + context.packed = packed; + context.conv1x1_kernel = conv1x1_kernel; + context.conv3x3_kernel = conv3x3_kernel; + worker_count = parallel_conv_worker_count(session, &context); + if (profile != NULL && worker_count < LW_EXECUTION_PROFILE_THREAD_HISTOGRAM_CAPACITY && + profile->conv_thread_histogram[worker_count] != UINT64_MAX) { + ++profile->conv_thread_histogram[worker_count]; + } + if (worker_count <= 1u) { + return LW_STATUS_UNSUPPORTED; + } + lw_thread_pool_run(session->thread_pool, worker_count, execute_parallel_conv_slice, &context); + for (worker_index = 0u; worker_index < worker_count; ++worker_index) { + if (context.statuses[worker_index] != LW_STATUS_OK) { + return context.statuses[worker_index]; + } + } + return LW_STATUS_OK; +} + +static uint32_t parallel_conv_transpose_worker_count( + const lw_session* session, const lw_parallel_conv_transpose_context* context) { + uint32_t maximum_workers; + uint32_t output_channels; + uint64_t work; + if (session->intra_op_thread_count <= 1u || context->groups != 1u || + context->input_dimensions[0] != 1 || context->output_dimensions[0] != 1 || + context->kernel[0] != 2 || context->kernel[1] != 2 || context->strides[0] != 2 || + context->strides[1] != 2 || context->dilations[0] != 1 || + context->dilations[1] != 1 || context->pads[0] != 0 || context->pads[1] != 0 || + context->pads[2] != 0 || context->pads[3] != 0 || + context->weight_dimensions[0] != context->input_dimensions[1] || + context->weight_dimensions[1] != context->output_dimensions[1] || + context->weight_dimensions[2] != 2 || context->weight_dimensions[3] != 2 || + context->output_dimensions[2] != context->input_dimensions[2] * 2 || + context->output_dimensions[3] != context->input_dimensions[3] * 2 || + (context->bias == NULL ? context->bias_count != 0u + : context->bias_count != + (uint32_t)context->output_dimensions[1])) { + return 1u; + } + output_channels = (uint32_t)context->output_dimensions[1]; + maximum_workers = output_channels; + if (maximum_workers > session->intra_op_thread_count) { + maximum_workers = session->intra_op_thread_count; + } + if (maximum_workers < 2u) { + return 1u; + } + work = multiply_saturated(output_channels, (uint32_t)context->input_dimensions[1]); + work = multiply_saturated(work, (uint32_t)context->input_dimensions[2]); + work = multiply_saturated(work, (uint32_t)context->input_dimensions[3]); + work = multiply_saturated(work, 4u); + return work >= LW_PARALLEL_CONV_TRANSPOSE_MIN_MULTIPLY_ADDS ? maximum_workers : 1u; +} + +static void execute_parallel_conv_transpose_slice(void* opaque, uint32_t worker_index, + uint32_t worker_count) { + lw_parallel_conv_transpose_context* context = + (lw_parallel_conv_transpose_context*)opaque; + const uint32_t output_channels = (uint32_t)context->output_dimensions[1]; + const uint32_t channel_begin = + (uint32_t)(((uint64_t)output_channels * worker_index) / worker_count); + const uint32_t channel_end = + (uint32_t)(((uint64_t)output_channels * (worker_index + 1u)) / worker_count); + if (lw_simd_level_is_avx2(context->simd_level)) { + lw_avx2_conv_transpose2x2_stride2_range_f32( + context->input, context->weights, context->bias, context->output, + context->input_dimensions, context->output_dimensions, channel_begin, channel_end); + } else if (lw_simd_level_is_neon(context->simd_level)) { + lw_neon_conv_transpose2x2_stride2_range_f32( + context->input, context->weights, context->bias, context->output, + context->input_dimensions, context->output_dimensions, channel_begin, channel_end); + } else if (lw_simd_level_is_sse2(context->simd_level)) { + lw_sse2_conv_transpose2x2_stride2_range_f32( + context->input, context->weights, context->bias, context->output, + context->input_dimensions, context->output_dimensions, channel_begin, channel_end); + } else { + lw_scalar_conv_transpose2x2_stride2_range_f32( + context->input, context->weights, context->bias, context->output, + context->input_dimensions, context->output_dimensions, channel_begin, channel_end); + } +} + +static lw_status dispatch_parallel_conv_transpose( + lw_session* session, const float* input, const float* weights, const float* bias, + uint32_t bias_count, float* output, const int32_t input_dimensions[4], + const int32_t weight_dimensions[4], const int32_t output_dimensions[4], + const int32_t kernel[2], const int32_t strides[2], const int32_t dilations[2], + const int32_t pads[4], uint32_t groups, lw_simd_level simd_level, + lw_execution_profile* profile) { + lw_parallel_conv_transpose_context context; + uint32_t worker_count; + memset(&context, 0, sizeof(context)); + context.input = input; + context.weights = weights; + context.bias = bias; + context.output = output; + memcpy(context.input_dimensions, input_dimensions, sizeof(context.input_dimensions)); + memcpy(context.weight_dimensions, weight_dimensions, sizeof(context.weight_dimensions)); + memcpy(context.output_dimensions, output_dimensions, sizeof(context.output_dimensions)); + memcpy(context.kernel, kernel, sizeof(context.kernel)); + memcpy(context.strides, strides, sizeof(context.strides)); + memcpy(context.dilations, dilations, sizeof(context.dilations)); + memcpy(context.pads, pads, sizeof(context.pads)); + context.groups = groups; + context.bias_count = bias_count; + context.simd_level = simd_level; + worker_count = parallel_conv_transpose_worker_count(session, &context); + if (profile != NULL && worker_count < LW_EXECUTION_PROFILE_THREAD_HISTOGRAM_CAPACITY && + profile->conv_transpose_thread_histogram[worker_count] != UINT64_MAX) { + ++profile->conv_transpose_thread_histogram[worker_count]; + } + if (worker_count <= 1u) { + return LW_STATUS_UNSUPPORTED; + } + lw_thread_pool_run(session->thread_pool, worker_count, + execute_parallel_conv_transpose_slice, &context); + return LW_STATUS_OK; +} + +static const lw_prepared_constant* prepared_constant_for_node( + const lw_session* session, uint32_t node_index, lw_execution_profile* profile) { +#if !defined(LW_EXPERIMENTAL_PREPARED_EXECUTION) + (void)profile; +#endif +#if defined(LW_EXPERIMENTAL_PREPARED_EXECUTION) + if (session->execution_nodes != NULL && node_index < session->execution_node_count) { + const lw_prepared_constant* prepared = + session->execution_nodes[node_index].prepared_constant; + if (profile != NULL && profile->prepared_binding_lookups != UINT64_MAX) { + ++profile->prepared_binding_lookups; + if (prepared != NULL) { + if (profile->prepared_binding_hits != UINT64_MAX) { + ++profile->prepared_binding_hits; + } + } else if (profile->prepared_binding_fallbacks != UINT64_MAX) { + ++profile->prepared_binding_fallbacks; + } + } + return prepared; + } +#endif + if (session->prepared_constants == NULL) { + return NULL; + } + return &session->prepared_constants[node_index]; +} + +static lw_status dispatch_node(lw_session* session, const uint8_t* node, uint32_t node_index, + uint32_t graph_input_index, const float* graph_input, + lw_simd_level simd_level, lw_execution_profile* profile) { + const lw_model* model = session->model; + const uint16_t op = lwm_read_u16(node); + const uint16_t input_count = lwm_read_u16(node + 2); + const uint16_t output_count = lwm_read_u16(node + 4); + const uint64_t param_offset = lwm_read_u64(node + 56); + const uint8_t* params = param_offset == 0u ? NULL : model->bytes + (size_t)param_offset; + const lw_runtime_tensor* input_tensors[LWM_V0_MAX_NODE_INPUTS]; + const float* inputs[LWM_V0_MAX_NODE_INPUTS]; + lw_runtime_tensor* output_tensor; + float* output; + uint32_t index; + const lw_prepared_constant* prepared_constant; + lw_packed_conv1x1_kernel_fn bound_kernel = NULL; + lw_packed_conv3x3_kernel_fn bound_conv3x3_kernel = NULL; + + if (output_count != 1u) { + return LW_STATUS_UNSUPPORTED; + } + for (index = 0u; index < input_count; ++index) { + uint32_t tensor_index = lwm_read_u32(node + 8u + index * 4u); + input_tensors[index] = &session->tensors[tensor_index]; + inputs[index] = tensor_input_data(session, tensor_index, graph_input_index, graph_input); + if (input_tensors[index]->dtype != LW_DTYPE_F32 || inputs[index] == NULL) { + return LW_STATUS_UNSUPPORTED; + } + } + index = lwm_read_u32(node + 40); + output_tensor = &session->tensors[index]; + output = tensor_output_data(session, index); + if (output_tensor->dtype != LW_DTYPE_F32 || output == NULL) { + return LW_STATUS_UNSUPPORTED; + } + prepared_constant = prepared_constant_for_node(session, node_index, profile); +#if defined(LW_EXPERIMENTAL_PREPARED_EXECUTION) + if (session->execution_nodes != NULL && node_index < session->execution_node_count && + session->execution_nodes[node_index].execution_kind == LW_BOUND_EXEC_CONV1X1_PACKED) { + bound_kernel = session->execution_nodes[node_index].data.conv1x1.kernel; + } else if (session->execution_nodes != NULL && + node_index < session->execution_node_count && + session->execution_nodes[node_index].execution_kind == LW_BOUND_EXEC_CONV3X3_PACKED) { + bound_conv3x3_kernel = session->execution_nodes[node_index].data.conv3x3.kernel; + } +#endif + + /* Parameters were structurally validated at model-load time. Each kernel + * still validates runtime-dependent shapes before reading tensor data. */ + switch (op) { + case LW_OP_CONV: { + int32_t kernel[2] = {lwm_read_i32(params + 8), lwm_read_i32(params + 12)}; + int32_t strides[2] = {lwm_read_i32(params + 16), lwm_read_i32(params + 20)}; + int32_t dilations[2] = {lwm_read_i32(params + 24), lwm_read_i32(params + 28)}; + int32_t pads[4] = {lwm_read_i32(params + 32), lwm_read_i32(params + 36), + lwm_read_i32(params + 40), lwm_read_i32(params + 44)}; + uint32_t groups = lwm_read_u32(params + 4); + lw_status parallel_status; + if (input_count != 2u && input_count != 3u) { + return LW_STATUS_INVALID_SHAPE; + } + { + if (prepared_constant != NULL && + prepared_constant->kind == LW_PREPARED_CONSTANT_CONV1X1_PACKED4) { + const float* packed_weights = + (const float*)(const void*)(session->packed_weights + + (size_t)prepared_constant->packed_weight_offset); + if (profile != NULL && profile->packed_conv1x1_invocations != UINT64_MAX) { + ++profile->packed_conv1x1_invocations; + } + parallel_status = dispatch_parallel_conv( + session, inputs[0], packed_weights, + input_count == 3u ? inputs[2] : NULL, output, + input_tensors[0]->dimensions, input_tensors[1]->dimensions, + output_tensor->dimensions, kernel, strides, dilations, pads, groups, 1u, + bound_kernel, NULL, profile); + if (parallel_status == LW_STATUS_OK) { + return LW_STATUS_OK; + } + if (bound_kernel != NULL) { + bound_kernel(inputs[0], packed_weights, + input_count == 3u ? inputs[2] : NULL, output, + input_tensors[0]->dimensions, output_tensor->dimensions); + } else { + lw_packed_conv1x1_f32( + inputs[0], packed_weights, input_count == 3u ? inputs[2] : NULL, + output, input_tensors[0]->dimensions, output_tensor->dimensions); + } + return LW_STATUS_OK; + } + } + { + if (prepared_constant != NULL && + prepared_constant->kind == LW_PREPARED_CONSTANT_CONV3X3_STRIDE2_PACKED8) { + const float* packed_weights = + (const float*)(const void*)(session->packed_weights + + (size_t)prepared_constant->packed_weight_offset); + if (profile != NULL && + profile->packed_conv3x3_stride2_invocations != UINT64_MAX) { + ++profile->packed_conv3x3_stride2_invocations; + } + parallel_status = dispatch_parallel_conv( + session, inputs[0], packed_weights, + input_count == 3u ? inputs[2] : NULL, output, + input_tensors[0]->dimensions, input_tensors[1]->dimensions, + output_tensor->dimensions, kernel, strides, dilations, pads, groups, + LW_PREPARED_CONSTANT_CONV3X3_STRIDE2_PACKED8, NULL, bound_conv3x3_kernel, profile); + if (parallel_status == LW_STATUS_OK) { + return LW_STATUS_OK; + } + if (bound_conv3x3_kernel != NULL) { + bound_conv3x3_kernel(inputs[0], packed_weights, + input_count == 3u ? inputs[2] : NULL, output, + input_tensors[0]->dimensions, output_tensor->dimensions); + } else { + lw_packed_conv3x3_stride2_pad1_f32( + inputs[0], packed_weights, input_count == 3u ? inputs[2] : NULL, + output, input_tensors[0]->dimensions, output_tensor->dimensions); + } + return LW_STATUS_OK; + } + } + if (profile != NULL && profile->unpacked_conv_invocations != UINT64_MAX) { + ++profile->unpacked_conv_invocations; + } + parallel_status = dispatch_parallel_conv( + session, inputs[0], inputs[1], input_count == 3u ? inputs[2] : NULL, output, + input_tensors[0]->dimensions, input_tensors[1]->dimensions, output_tensor->dimensions, + kernel, strides, dilations, pads, groups, 0u, NULL, NULL, profile); + if (parallel_status == LW_STATUS_OK) { + return LW_STATUS_OK; + } + return lw_scalar_conv2d_f32( + inputs[0], inputs[1], input_count == 3u ? inputs[2] : NULL, + input_count == 3u ? (uint32_t)tensor_element_count(input_tensors[2]) : 0u, output, + input_tensors[0]->dimensions, input_tensors[1]->dimensions, output_tensor->dimensions, + kernel, strides, dilations, pads, groups); + } + case LW_OP_ADD: + case LW_OP_MUL: + case LW_OP_DIV: + case LW_OP_SUB: + case LW_OP_POW: { + lw_scalar_binary_op operation = + op == LW_OP_ADD ? LW_SCALAR_BINARY_ADD + : (op == LW_OP_MUL ? LW_SCALAR_BINARY_MUL + : (op == LW_OP_DIV ? LW_SCALAR_BINARY_DIV + : (op == LW_OP_SUB + ? LW_SCALAR_BINARY_SUB + : LW_SCALAR_BINARY_POW))); + if (input_count != 2u) { + return LW_STATUS_INVALID_SHAPE; + } + return lw_scalar_binary_f32(operation, inputs[0], input_tensors[0]->rank, + input_tensors[0]->dimensions, inputs[1], input_tensors[1]->rank, + input_tensors[1]->dimensions, output, output_tensor->rank, + output_tensor->dimensions); + } + case LW_OP_ERF: { + uint64_t element_count; + if (input_count != 1u) { + return LW_STATUS_INVALID_SHAPE; + } + element_count = tensor_element_count(input_tensors[0]); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_erf_f32(inputs[0], output, element_count); + return LW_STATUS_OK; + } +#if defined(__EMSCRIPTEN__) + if (lw_simd_level_is_sse2(simd_level)) { + lw_wasm128_erf_f32(inputs[0], output, element_count); + return LW_STATUS_OK; + } +#endif + return lw_scalar_erf_f32(inputs[0], output, element_count); + } + case LW_OP_HARD_SIGMOID: { + float alpha; + float beta; + uint64_t element_count; + if (input_count != 1u) { + return LW_STATUS_INVALID_SHAPE; + } + alpha = read_f32(params + 4); + beta = read_f32(params + 8); + element_count = tensor_element_count(input_tensors[0]); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_hard_sigmoid_exact_f32(inputs[0], output, element_count, alpha, beta); + return LW_STATUS_OK; + } + return lw_scalar_hard_sigmoid_f32(inputs[0], output, element_count, alpha, beta); + } + case LW_OP_BATCH_NORMALIZATION: + return input_count == 5u + ? lw_scalar_batch_normalization_f32( + inputs[0], inputs[1], inputs[2], inputs[3], inputs[4], + (uint32_t)tensor_element_count(input_tensors[1]), read_f32(params + 4), + output, input_tensors[0]->rank, input_tensors[0]->dimensions) + : LW_STATUS_INVALID_SHAPE; + case LW_OP_REDUCE_MEAN: { + int32_t axes[LW_MAX_DIMS]; + uint32_t axes_count = lwm_read_u16(params + 2); + if (input_count != 1u) { + return LW_STATUS_INVALID_SHAPE; + } + for (index = 0u; index < axes_count; ++index) { + axes[index] = lwm_read_i32(params + 12u + index * 4u); + } + return lw_scalar_reduce_mean_f32(inputs[0], output, input_tensors[0]->rank, + input_tensors[0]->dimensions, axes_count, axes, + lwm_read_u32(params + 4), lwm_read_u32(params + 8), + output_tensor->rank, output_tensor->dimensions); + } + case LW_OP_RELU: { + uint64_t element_count; + if (input_count != 1u) { + return LW_STATUS_INVALID_SHAPE; + } + element_count = tensor_element_count(input_tensors[0]); + if (lw_simd_level_is_avx2(simd_level)) { + lw_avx2_relu_f32(inputs[0], output, element_count); + return LW_STATUS_OK; + } + return lw_scalar_relu_f32(inputs[0], output, element_count); + } + case LW_OP_AVERAGE_POOL: { + int32_t kernel[2] = {lwm_read_i32(params + 8), lwm_read_i32(params + 12)}; + int32_t strides[2] = {lwm_read_i32(params + 16), lwm_read_i32(params + 20)}; + int32_t pads[4] = {lwm_read_i32(params + 24), lwm_read_i32(params + 28), + lwm_read_i32(params + 32), lwm_read_i32(params + 36)}; + if (input_count != 1u) { + return LW_STATUS_INVALID_SHAPE; + } + return lw_scalar_average_pool2d_f32(inputs[0], output, input_tensors[0]->dimensions, + output_tensor->dimensions, kernel, strides, pads, + lwm_read_u32(params + 40), lwm_read_u32(params + 44)); + } + case LW_OP_SQUEEZE: + case LW_OP_TRANSPOSE: + case LW_OP_UNSQUEEZE: { + int32_t axes[LW_MAX_DIMS]; + uint32_t axes_count = lwm_read_u16(params + 2); + if (input_count != 1u) { + return LW_STATUS_INVALID_SHAPE; + } + for (index = 0u; index < axes_count; ++index) { + axes[index] = lwm_read_i32(params + 4u + index * 4u); + } + if (op == LW_OP_SQUEEZE) { + return lw_scalar_squeeze_f32(inputs[0], output, input_tensors[0]->rank, + input_tensors[0]->dimensions, axes_count, axes, + output_tensor->rank, output_tensor->dimensions); + } + if (op == LW_OP_UNSQUEEZE) { + return lw_scalar_unsqueeze_f32(inputs[0], output, input_tensors[0]->rank, + input_tensors[0]->dimensions, axes_count, axes, + output_tensor->rank, output_tensor->dimensions); + } + return lw_scalar_transpose_f32(inputs[0], output, input_tensors[0]->rank, + input_tensors[0]->dimensions, axes_count, axes, + output_tensor->dimensions); + } + case LW_OP_MATMUL: { + uint64_t batch_count = 1u; + uint32_t rank = input_tensors[0]->rank; + if (input_count != 2u || rank < 2u || input_tensors[1]->rank < 2u) { + return LW_STATUS_UNSUPPORTED; + } + if (input_tensors[1]->rank != 2u) { + if (profile != NULL && profile->unpacked_matmul_invocations != UINT64_MAX) { + ++profile->unpacked_matmul_invocations; + } + return lw_scalar_matmul_f32(inputs[0], inputs[1], output, rank, + input_tensors[0]->dimensions, input_tensors[1]->rank, + input_tensors[1]->dimensions, output_tensor->rank, + output_tensor->dimensions); + } + for (index = 0u; index + 2u < rank; ++index) { + batch_count *= (uint32_t)input_tensors[0]->dimensions[index]; + } + if (batch_count > UINT32_MAX) { + return LW_STATUS_OUT_OF_BOUNDS; + } + { + if (lw_simd_level_is_avx2(simd_level) && prepared_constant != NULL && + prepared_constant->kind == LW_PREPARED_CONSTANT_MATMUL_PACKED16) { + if (profile != NULL && profile->packed_matmul_invocations != UINT64_MAX) { + ++profile->packed_matmul_invocations; + } + const float* packed_weights = + (const float*)(const void*)(session->packed_weights + + (size_t)prepared_constant->packed_weight_offset); + lw_avx2_packed_matmul_shared_f32(inputs[0], packed_weights, output, + (uint32_t)batch_count, + (uint32_t)input_tensors[0]->dimensions[rank - 2u], + (uint32_t)input_tensors[0]->dimensions[rank - 1u], + (uint32_t)input_tensors[1]->dimensions[1]); + return LW_STATUS_OK; + } + } + if (profile != NULL && profile->unpacked_matmul_invocations != UINT64_MAX) { + ++profile->unpacked_matmul_invocations; + } + return lw_matmul_shared_f32(inputs[0], inputs[1], output, (uint32_t)batch_count, + (uint32_t)input_tensors[0]->dimensions[rank - 2u], + (uint32_t)input_tensors[0]->dimensions[rank - 1u], + (uint32_t)input_tensors[1]->dimensions[1]); + } + case LW_OP_SOFTMAX: + return input_count == 1u + ? lw_scalar_softmax_f32(inputs[0], output, input_tensors[0]->rank, + input_tensors[0]->dimensions, lwm_read_i32(params + 4)) + : LW_STATUS_INVALID_SHAPE; + case LW_OP_RESHAPE: + return input_count == 1u + ? lw_scalar_reshape_f32(inputs[0], output, input_tensors[0]->rank, + input_tensors[0]->dimensions, output_tensor->rank, + output_tensor->dimensions) + : LW_STATUS_INVALID_SHAPE; + case LW_OP_CONCAT: { + uint32_t ranks[LWM_V0_MAX_NODE_INPUTS]; + const int32_t* dimensions[LWM_V0_MAX_NODE_INPUTS]; + for (index = 0u; index < input_count; ++index) { + ranks[index] = input_tensors[index]->rank; + dimensions[index] = input_tensors[index]->dimensions; + } + return lw_scalar_concat_f32(inputs, input_count, ranks, dimensions, output, + output_tensor->rank, output_tensor->dimensions, + lwm_read_i32(params + 4)); + } + case LW_OP_CONV_TRANSPOSE: { + int32_t kernel[2] = {lwm_read_i32(params + 8), lwm_read_i32(params + 12)}; + int32_t strides[2] = {lwm_read_i32(params + 16), lwm_read_i32(params + 20)}; + int32_t dilations[2] = {lwm_read_i32(params + 24), lwm_read_i32(params + 28)}; + int32_t pads[4] = {lwm_read_i32(params + 32), lwm_read_i32(params + 36), + lwm_read_i32(params + 40), lwm_read_i32(params + 44)}; + if (input_count != 2u && input_count != 3u) { + return LW_STATUS_INVALID_SHAPE; + } + if (dispatch_parallel_conv_transpose( + session, inputs[0], inputs[1], input_count == 3u ? inputs[2] : NULL, + input_count == 3u ? (uint32_t)tensor_element_count(input_tensors[2]) : 0u, + output, input_tensors[0]->dimensions, input_tensors[1]->dimensions, + output_tensor->dimensions, kernel, strides, dilations, pads, + lwm_read_u32(params + 4), simd_level, profile) == LW_STATUS_OK) { + return LW_STATUS_OK; + } + return lw_scalar_conv_transpose2d_f32( + inputs[0], inputs[1], input_count == 3u ? inputs[2] : NULL, + input_count == 3u ? (uint32_t)tensor_element_count(input_tensors[2]) : 0u, output, + input_tensors[0]->dimensions, input_tensors[1]->dimensions, output_tensor->dimensions, + kernel, strides, dilations, pads, lwm_read_u32(params + 4)); + } + case LW_OP_MAX_POOL: { + int32_t kernel[2] = {lwm_read_i32(params + 8), lwm_read_i32(params + 12)}; + int32_t strides[2] = {lwm_read_i32(params + 16), lwm_read_i32(params + 20)}; + int32_t pads[4] = {lwm_read_i32(params + 24), lwm_read_i32(params + 28), + lwm_read_i32(params + 32), lwm_read_i32(params + 36)}; + return input_count == 1u + ? lw_scalar_max_pool2d_f32(inputs[0], output, input_tensors[0]->dimensions, + output_tensor->dimensions, kernel, strides, pads, + lwm_read_u32(params + 40)) + : LW_STATUS_INVALID_SHAPE; + } + case LW_OP_RESIZE: { + float scales[LW_MAX_DIMS]; + uint32_t scale_count = lwm_read_u16(params + 2); + if (input_count != 1u || scale_count != input_tensors[0]->rank) { + return LW_STATUS_INVALID_SHAPE; + } + for (index = 0u; index < scale_count; ++index) { + scales[index] = read_f32(params + 4u + index * 4u); + } + return lw_scalar_resize_nearest_f32(inputs[0], output, input_tensors[0]->rank, + input_tensors[0]->dimensions, output_tensor->dimensions, + scales); + } + case LW_OP_SIGMOID: + return input_count == 1u ? lw_scalar_sigmoid_f32(inputs[0], output, + tensor_element_count(input_tensors[0])) + : LW_STATUS_INVALID_SHAPE; + case LW_OP_SQRT: + return input_count == 1u ? lw_scalar_sqrt_f32(inputs[0], output, + tensor_element_count(input_tensors[0])) + : LW_STATUS_INVALID_SHAPE; + case LW_OP_SLICE: { + int32_t starts[LW_MAX_DIMS]; + int32_t ends[LW_MAX_DIMS]; + int32_t axes[LW_MAX_DIMS]; + int32_t steps[LW_MAX_DIMS]; + uint32_t slice_count = lwm_read_u16(params + 2u); + uint32_t slice_index; + if (input_count != 1u || slice_count == 0u || slice_count > LW_MAX_DIMS) { + return LW_STATUS_INVALID_SHAPE; + } + for (slice_index = 0u; slice_index < slice_count; ++slice_index) { + starts[slice_index] = lwm_read_i32(params + 4u + slice_index * 4u); + ends[slice_index] = lwm_read_i32(params + 36u + slice_index * 4u); + axes[slice_index] = lwm_read_i32(params + 68u + slice_index * 4u); + steps[slice_index] = lwm_read_i32(params + 100u + slice_index * 4u); + } + return lw_scalar_slice_f32(inputs[0], output, input_tensors[0]->rank, + input_tensors[0]->dimensions, output_tensor->dimensions, + slice_count, starts, ends, axes, steps); + } + default: + return LW_STATUS_UNSUPPORTED; + } +} + +const float* lw_executor_tensor_input_data(const lw_session* session, + uint32_t tensor_index, + uint32_t graph_input_index, + const float* graph_input) { + if (session == NULL || session->model == NULL || session->tensors == NULL || + tensor_index >= session->model->info.tensor_count) { + return NULL; + } + return tensor_input_data(session, tensor_index, graph_input_index, graph_input); +} + +float* lw_executor_tensor_output_data(lw_session* session, uint32_t tensor_index) { + if (session == NULL || session->model == NULL || session->tensors == NULL || + tensor_index >= session->model->info.tensor_count) { + return NULL; + } + return tensor_output_data(session, tensor_index); +} + +lw_status lw_executor_dispatch_node_f32(lw_session* session, uint32_t node_index, + uint32_t graph_input_index, + const float* graph_input, + lw_execution_profile* profile) { + const uint8_t* node; + if (session == NULL || session->model == NULL || + node_index >= session->model->info.node_count) { + return LW_STATUS_INVALID_ARGUMENT; + } + node = session->model->bytes + (size_t)session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + return dispatch_node(session, node, node_index, graph_input_index, graph_input, + session->cpu.simd, profile); +} +static int match_avx2_gelu(lw_session* session, uint32_t node_index, uint32_t graph_input_index, + const float* graph_input, const float** gelu_input, float** gelu_output, + uint64_t* element_count) { + lw_fused_gelu_match match; + uint32_t index; + const lw_runtime_tensor* source; + const lw_runtime_tensor* output; +#if defined(LW_EXPERIMENTAL_FUSION_MEMORY) + if (!lw_execution_plan_is_fused_gelu_start(session, node_index)) { + return 0; + } +#endif + if (!lw_match_fused_gelu(session, node_index, &match)) { + return 0; + } + /* Every skipped temporary must be private to this chain. Otherwise a + * later node could observe a value that the fused execution never wrote. */ + for (index = 0u; index < 4u; ++index) { + if (session->tensors[match.outputs[index]].last_use_node != + (int32_t)(node_index + index + 1u)) { + return 0; + } + } + source = &session->tensors[match.inputs[0][0]]; + output = &session->tensors[match.outputs[4]]; + if (source->dtype != LW_DTYPE_F32 || output->dtype != LW_DTYPE_F32 || source->byte_size == 0u || + source->byte_size != output->byte_size) { + return 0; + } + for (index = 0u; index < 4u; ++index) { + const lw_runtime_tensor* temporary = &session->tensors[match.outputs[index]]; + if (temporary->dtype != LW_DTYPE_F32 || temporary->byte_size != source->byte_size) { + return 0; + } + } + *gelu_input = tensor_input_data(session, match.inputs[0][0], graph_input_index, graph_input); + *gelu_output = tensor_output_data(session, match.outputs[4]); + *element_count = source->byte_size / sizeof(float); + return *gelu_input != NULL && *gelu_output != NULL; +} + +static uint32_t profile_conv_class(const lw_session* session, const uint8_t* node) { + const lw_model* model = session->model; + uint64_t param_offset = lwm_read_u64(node + 56); + const uint8_t* params = model->bytes + (size_t)param_offset; + const lw_runtime_tensor* input = &session->tensors[lwm_read_u32(node + 8u)]; + const lw_runtime_tensor* weights = &session->tensors[lwm_read_u32(node + 12u)]; + int32_t kernel_height = lwm_read_i32(params + 8u); + int32_t kernel_width = lwm_read_i32(params + 12u); + int32_t stride_height = lwm_read_i32(params + 16u); + int32_t stride_width = lwm_read_i32(params + 20u); + uint32_t group = lwm_read_u32(params + 4u); + + if (kernel_height == 1 && kernel_width == 1) { + return LW_EXECUTION_PROFILE_CONV_1X1; + } + if (kernel_height == 3 && kernel_width == 3 && input->rank == 4u && weights->rank == 4u && + input->dimensions[1] > 0 && group == (uint32_t)input->dimensions[1] && + weights->dimensions[0] == input->dimensions[1]) { + return LW_EXECUTION_PROFILE_CONV_DEPTHWISE_3X3; + } + if (kernel_height == 3 && kernel_width == 3 && stride_height == 2 && stride_width == 2) { + return LW_EXECUTION_PROFILE_CONV_STRIDE2_3X3; + } + if (kernel_height == 3 && kernel_width == 3) { + return LW_EXECUTION_PROFILE_CONV_3X3; + } + return LW_EXECUTION_PROFILE_CONV_OTHER; +} + +static void profile_fused_gelu(lw_execution_profile* profile, const lw_session* session, + uint32_t first_node_index, uint64_t elapsed) { + uint32_t offset; + if (profile == NULL) { + return; + } + if (profile->fused_gelu_invocations != UINT64_MAX) { + ++profile->fused_gelu_invocations; + } + /* The public profile schema has no fused-GELU operator. Attribute the + * combined work to Erf and retain one invocation for every semantic node; + * one-nanosecond placeholders keep existing per-node coverage checks useful. */ + for (offset = 0u; offset < 5u; ++offset) { + uint32_t node_index = first_node_index + offset; + const uint8_t* node = session->model->bytes + (size_t)session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + uint32_t operation = (uint32_t)lwm_read_u16(node); + uint64_t node_elapsed = offset == 1u ? (elapsed == 0u ? 1u : elapsed) : 1u; + if (operation < LW_EXECUTION_PROFILE_OPERATOR_CAPACITY && + profile->operator_nanoseconds[operation] <= UINT64_MAX - node_elapsed && + profile->operator_invocations[operation] != UINT64_MAX) { + profile->operator_nanoseconds[operation] += node_elapsed; + profile->operator_invocations[operation] += 1u; + } + if (node_index < LW_EXECUTION_PROFILE_NODE_CAPACITY && + profile->node_nanoseconds[node_index] <= UINT64_MAX - node_elapsed && + profile->node_invocations[node_index] != UINT64_MAX) { + profile->node_nanoseconds[node_index] += node_elapsed; + profile->node_invocations[node_index] += 1u; + } + } +} + +#if defined(LW_EXPERIMENTAL_PREPARED_EXECUTION) +static lw_status execute_bound_conv1x1(lw_session* session, const lw_bound_node* bound, + uint32_t graph_input_index, const float* graph_input, + lw_execution_profile* profile) { + const lw_bound_conv1x1* conv = &bound->data.conv1x1; + const lw_runtime_tensor* input_tensor = &session->tensors[conv->input_index]; + const lw_runtime_tensor* output_tensor = &session->tensors[conv->output_index]; + const float* input = tensor_input_data(session, conv->input_index, graph_input_index, graph_input); + const float* bias = conv->bias_index == UINT32_MAX + ? NULL + : tensor_input_data(session, conv->bias_index, graph_input_index, graph_input); + float* output = tensor_output_data(session, conv->output_index); + int32_t weight_dimensions[4] = {output_tensor->dimensions[1], input_tensor->dimensions[1], 1, 1}; + int32_t kernel[2] = {1, 1}; + int32_t strides[2] = {1, 1}; + int32_t dilations[2] = {1, 1}; + int32_t pads[4] = {0, 0, 0, 0}; + lw_status status; + + if (input == NULL || output == NULL || conv->kernel == NULL || + input_tensor->dtype != LW_DTYPE_F32 || output_tensor->dtype != LW_DTYPE_F32 || + (conv->bias_index != UINT32_MAX && bias == NULL)) { + return LW_STATUS_UNSUPPORTED; + } + status = dispatch_parallel_conv( + session, input, conv->packed_weights, bias, output, input_tensor->dimensions, + weight_dimensions, output_tensor->dimensions, kernel, strides, dilations, pads, 1u, + LW_PREPARED_CONSTANT_CONV1X1_PACKED4, conv->kernel, NULL, profile); + if (status == LW_STATUS_OK) { + return LW_STATUS_OK; + } + conv->kernel(input, conv->packed_weights, bias, output, + input_tensor->dimensions, output_tensor->dimensions); + return LW_STATUS_OK; +} +static lw_status execute_bound_conv3x3(lw_session* session, const lw_bound_node* bound, + uint32_t graph_input_index, const float* graph_input, + lw_execution_profile* profile) { + const lw_bound_conv3x3* conv = &bound->data.conv3x3; + const lw_runtime_tensor* input_tensor = &session->tensors[conv->input_index]; + const lw_runtime_tensor* output_tensor = &session->tensors[conv->output_index]; + const float* input = tensor_input_data(session, conv->input_index, graph_input_index, graph_input); + const float* bias = conv->bias_index == UINT32_MAX + ? NULL + : tensor_input_data(session, conv->bias_index, graph_input_index, graph_input); + float* output = tensor_output_data(session, conv->output_index); + int32_t weight_dimensions[4] = {output_tensor->dimensions[1], input_tensor->dimensions[1], 3, 3}; + int32_t kernel[2] = {3, 3}; + int32_t strides[2] = {2, 2}; + int32_t dilations[2] = {1, 1}; + int32_t pads[4] = {1, 1, 1, 1}; + lw_status status; + + if (input == NULL || output == NULL || conv->kernel == NULL || + input_tensor->dtype != LW_DTYPE_F32 || output_tensor->dtype != LW_DTYPE_F32 || + (conv->bias_index != UINT32_MAX && bias == NULL)) { + return LW_STATUS_UNSUPPORTED; + } + status = dispatch_parallel_conv( + session, input, conv->packed_weights, bias, output, input_tensor->dimensions, + weight_dimensions, output_tensor->dimensions, kernel, strides, dilations, pads, 1u, + LW_PREPARED_CONSTANT_CONV3X3_STRIDE2_PACKED8, NULL, conv->kernel, profile); + if (status == LW_STATUS_OK) { + return LW_STATUS_OK; + } + conv->kernel(input, conv->packed_weights, bias, output, + input_tensor->dimensions, output_tensor->dimensions); + return LW_STATUS_OK; +} + +#endif + +/* Execute one semantic node through the strongest legacy path available. The + * Hybrid executor uses this bridge whenever an NHWC implementation is not + * proven for the current node. */ +lw_status lw_executor_execute_best_node_f32(lw_session* session, uint32_t node_index, + uint32_t graph_input_index, + const float* graph_input, + lw_execution_profile* profile, + uint32_t* consumed_nodes) { + const uint8_t* node; + uint32_t operation; + if (consumed_nodes != NULL) { + *consumed_nodes = 1u; + } + if (session == NULL || session->model == NULL || + node_index >= session->model->info.node_count) { + return LW_STATUS_INVALID_ARGUMENT; + } + node = session->model->bytes + (size_t)session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + operation = (uint32_t)lwm_read_u16(node); + if (lw_simd_level_is_avx2(session->cpu.simd) && operation == LW_OP_DIV) { + const float* gelu_input; + float* gelu_output; + uint64_t element_count; + if (match_avx2_gelu(session, node_index, graph_input_index, graph_input, + &gelu_input, &gelu_output, &element_count)) { + lw_avx2_gelu_f32(gelu_input, gelu_output, element_count); + if (consumed_nodes != NULL) { + *consumed_nodes = 5u; + } + return LW_STATUS_OK; + } + } +#if defined(LW_EXPERIMENTAL_PREPARED_EXECUTION) + if (session->execution_nodes != NULL && node_index < session->execution_node_count) { + const lw_bound_node* bound = &session->execution_nodes[node_index]; + if (bound->execution_kind == LW_BOUND_EXEC_CONV1X1_PACKED) { + return execute_bound_conv1x1(session, bound, graph_input_index, graph_input, profile); + } + if (bound->execution_kind == LW_BOUND_EXEC_CONV3X3_PACKED) { + return execute_bound_conv3x3(session, bound, graph_input_index, graph_input, profile); + } + } +#endif + return dispatch_node(session, node, node_index, graph_input_index, graph_input, + session->cpu.simd, profile); +} +static lw_status execute_session_nodes_f32(lw_session* session, const float* input, + uint64_t input_element_count, uint32_t node_limit, + lw_execution_profile* profile, lw_error* error) { + const lw_model* model; + uint32_t graph_input_index; + uint32_t node_index; + lw_simd_level simd_level; + lw_status status; + char message[LW_ERROR_MESSAGE_CAPACITY]; + + if (session == NULL || input == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "session and input are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + model = session->model; + /* The session stores one CPU capability snapshot for the complete run. */ + simd_level = session->cpu.simd; + if (model->info.input_count != 1u || model->info.output_count != 1u) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "private executor currently requires one input and one output"); + return LW_STATUS_UNSUPPORTED; + } + graph_input_index = lwm_read_u32(model->bytes + (size_t)model->input_offset); + if (session->tensors[graph_input_index].dtype != LW_DTYPE_F32 || + input_element_count != tensor_element_count(&session->tensors[graph_input_index]) || + node_limit > model->info.node_count) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "input shape or node limit is invalid"); + return LW_STATUS_INVALID_SHAPE; + } + profile_layout_candidate_nodes(profile, session, node_limit); + for (node_index = 0u; node_index < node_limit; ++node_index) { + const uint8_t* node = + model->bytes + (size_t)model->node_offset + (size_t)node_index * LWM_V0_NODE_SIZE; + uint32_t operation = (uint32_t)lwm_read_u16(node); + uint64_t started = 0u; + if (lw_simd_level_is_avx2(simd_level) && operation == LW_OP_DIV) { + const float* gelu_input; + float* gelu_output; + uint64_t gelu_element_count; + if (match_avx2_gelu(session, node_index, graph_input_index, input, &gelu_input, + &gelu_output, &gelu_element_count)) { + uint64_t elapsed = 0u; + if (profile != NULL) { + started = profile->clock(profile->clock_context); + } + lw_avx2_gelu_f32(gelu_input, gelu_output, gelu_element_count); + if (profile != NULL) { + uint64_t finished = profile->clock(profile->clock_context); + if (finished >= started) { + elapsed = finished - started; + } + } + profile_fused_gelu(profile, session, node_index, elapsed); + node_index += 4u; + continue; + } + } + if (profile != NULL) { + started = profile->clock(profile->clock_context); + } +#if defined(LW_EXPERIMENTAL_PREPARED_EXECUTION) + if (session->execution_nodes != NULL && node_index < session->execution_node_count && + session->execution_nodes[node_index].execution_kind == LW_BOUND_EXEC_CONV1X1_PACKED) { + status = execute_bound_conv1x1(session, &session->execution_nodes[node_index], + graph_input_index, input, profile); + } else if (session->execution_nodes != NULL && + node_index < session->execution_node_count && + session->execution_nodes[node_index].execution_kind == LW_BOUND_EXEC_CONV3X3_PACKED) { + status = execute_bound_conv3x3(session, &session->execution_nodes[node_index], + graph_input_index, input, profile); + } else { + status = dispatch_node(session, node, node_index, graph_input_index, input, + simd_level, profile); + if (profile != NULL && profile->generic_node_invocations != UINT64_MAX) { + ++profile->generic_node_invocations; + } + } + if (profile != NULL && session->execution_nodes != NULL && + node_index < session->execution_node_count && + session->execution_nodes[node_index].execution_kind != LW_BOUND_EXEC_GENERIC) { + if (profile->prepared_binding_lookups != UINT64_MAX) { + ++profile->prepared_binding_lookups; + } + if (profile->prepared_binding_hits != UINT64_MAX) { + ++profile->prepared_binding_hits; + } + if (profile->prepared_node_invocations != UINT64_MAX) { + ++profile->prepared_node_invocations; + } + if (session->execution_nodes[node_index].execution_kind == LW_BOUND_EXEC_CONV1X1_PACKED) { + if (profile->prepared_conv1x1_invocations != UINT64_MAX) { + ++profile->prepared_conv1x1_invocations; + } + if (profile->packed_conv1x1_invocations != UINT64_MAX) { + ++profile->packed_conv1x1_invocations; + } + } else { + if (profile->prepared_conv3x3_invocations != UINT64_MAX) { + ++profile->prepared_conv3x3_invocations; + } + if (profile->packed_conv3x3_stride2_invocations != UINT64_MAX) { + ++profile->packed_conv3x3_stride2_invocations; + } + } + } +#else + status = + dispatch_node(session, node, node_index, graph_input_index, input, simd_level, profile); +#endif + if (profile != NULL && operation < LW_EXECUTION_PROFILE_OPERATOR_CAPACITY) { + uint64_t finished = profile->clock(profile->clock_context); + if (finished >= started) { + uint64_t elapsed = finished - started; + if (profile->operator_nanoseconds[operation] <= UINT64_MAX - elapsed && + profile->operator_invocations[operation] != UINT64_MAX) { + profile->operator_nanoseconds[operation] += elapsed; + profile->operator_invocations[operation] += 1u; + } + if (node_index < LW_EXECUTION_PROFILE_NODE_CAPACITY && + profile->node_nanoseconds[node_index] <= UINT64_MAX - elapsed && + profile->node_invocations[node_index] != UINT64_MAX) { + profile->node_nanoseconds[node_index] += elapsed; + profile->node_invocations[node_index] += 1u; + } + if (operation == LW_OP_CONV) { + uint32_t conv_class = profile_conv_class(session, node); + if (profile->conv_class_nanoseconds[conv_class] <= UINT64_MAX - elapsed && + profile->conv_class_invocations[conv_class] != UINT64_MAX) { + profile->conv_class_nanoseconds[conv_class] += elapsed; + profile->conv_class_invocations[conv_class] += 1u; + } + } + if (operation == LW_OP_TRANSPOSE && status == LW_STATUS_OK) { + uint32_t output_index = lwm_read_u32(node + 40u); + uint64_t bytes = output_index < session->model->info.tensor_count + ? session->tensors[output_index].byte_size + : 0u; + profile_add_layout_value(&profile->layout_transform_nanoseconds, elapsed); + if (profile->layout_transform_invocations != UINT64_MAX) { + ++profile->layout_transform_invocations; + } + profile_add_layout_value(&profile->layout_transform_bytes, bytes); + } + } + } + if (status != LW_STATUS_OK) { +#if defined(_MSC_VER) + (void)sprintf_s(message, sizeof(message), "node %u (operator %u) failed: %s", + node_index, (uint32_t)lwm_read_u16(node), lw_status_string(status)); +#else + (void)snprintf(message, sizeof(message), "node %u (operator %u) failed: %s", node_index, + (uint32_t)lwm_read_u16(node), lw_status_string(status)); +#endif + lw_set_error(error, status, message); + return status; + } + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +static lw_status execute_session_f32(lw_session* session, const float* input, + uint64_t input_element_count, float* output, + uint64_t output_element_count, lw_execution_profile* profile, + lw_error* error) { + uint32_t graph_output_index; + lw_status status; + if (session == NULL || input == NULL || output == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "session, input, and output are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + graph_output_index = lwm_read_u32(session->model->bytes + + (size_t)session->model->output_offset); + if (session->tensors[graph_output_index].dtype != LW_DTYPE_F32 || + output_element_count != tensor_element_count(&session->tensors[graph_output_index])) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "output element count does not match the session"); + return LW_STATUS_INVALID_SHAPE; + } + status = execute_session_nodes_f32(session, input, input_element_count, + session->model->info.node_count, profile, error); + if (status != LW_STATUS_OK) { + return status; + } + memcpy(output, tensor_output_data(session, graph_output_index), + (size_t)session->tensors[graph_output_index].byte_size); + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +static int ctc_greedy_tail(const lw_session* session, uint32_t time_steps, + uint32_t class_count, uint32_t* logits_index) { + const lw_model* model; + const uint8_t* node; + const uint8_t* params; + const lw_runtime_tensor* logits; + const lw_runtime_tensor* output; + uint32_t graph_output_index; + uint32_t input_index; + uint32_t output_index; + uint32_t index; + int32_t axis; + if (session == NULL || time_steps == 0u || class_count == 0u) { + return 0; + } + model = session->model; + if (model->info.input_count != 1u || model->info.output_count != 1u || + model->info.node_count == 0u) { + return 0; + } + /* This is deliberately a structural check rather than a model-name + * special case. Unsupported or multi-output graphs retain the generic + * executor and complete probability tensor. */ + node = model->bytes + (size_t)model->node_offset + + (size_t)(model->info.node_count - 1u) * LWM_V0_NODE_SIZE; + if (lwm_read_u16(node) != LW_OP_SOFTMAX || lwm_read_u16(node + 2u) != 1u || + lwm_read_u16(node + 4u) != 1u) { + return 0; + } + input_index = lwm_read_u32(node + 8u); + output_index = lwm_read_u32(node + 40u); + graph_output_index = lwm_read_u32(model->bytes + (size_t)model->output_offset); + if (output_index != graph_output_index) { + return 0; + } + logits = &session->tensors[input_index]; + output = &session->tensors[output_index]; + if (logits->dtype != LW_DTYPE_F32 || output->dtype != LW_DTYPE_F32 || logits->rank < 2u || + logits->rank != output->rank || logits->last_use_node != (int32_t)(model->info.node_count - 1u) || + tensor_element_count(output) != (uint64_t)time_steps * class_count || + logits->dimensions[logits->rank - 1u] != (int32_t)class_count) { + return 0; + } + for (index = 0u; index < logits->rank; ++index) { + if (logits->dimensions[index] != output->dimensions[index]) { + return 0; + } + } + params = model->bytes + (size_t)lwm_read_u64(node + 56u); + axis = lwm_read_i32(params + 4u); + if (axis < 0) { + axis += (int32_t)logits->rank; + } + if (axis != (int32_t)logits->rank - 1) { + return 0; + } + if (logits_index != NULL) { + *logits_index = input_index; + } + return 1; +} + +int lw_session_supports_ctc_greedy_f32(const lw_session* session, uint32_t time_steps, + uint32_t class_count) { + return ctc_greedy_tail(session, time_steps, class_count, NULL); +} + +typedef struct lw_packed_ctc_projection { + uint32_t matmul_node_index; + uint32_t add_node_index; + const float* activation; + const float* packed_weights; + const float* bias; + float* logits; + uint32_t rows; + uint32_t inner_dimension; + uint32_t columns; +} lw_packed_ctc_projection; + +static int match_packed_ctc_projection(lw_session* session, const float* graph_input, + uint32_t logits_index, uint32_t time_steps, + uint32_t class_count, lw_execution_profile* profile, + lw_packed_ctc_projection* projection) { + const lw_model* model = session->model; + uint32_t matmul_node_index; + uint32_t add_node_index; + const uint8_t* matmul_node; + const uint8_t* add_node; + uint32_t matmul_input_index; + uint32_t weights_index; + uint32_t matmul_output_index; + uint32_t bias_index; + const lw_runtime_tensor* activation; + const lw_runtime_tensor* weights; + const lw_runtime_tensor* matmul_output; + const lw_runtime_tensor* bias; + const lw_prepared_constant* prepared; + uint32_t graph_input_index; + if (projection == NULL || model->info.node_count < 3u || + !lw_simd_level_is_avx2(session->cpu.simd) || session->prepared_constants == NULL || + session->packed_weights == NULL) { + return 0; + } + matmul_node_index = model->info.node_count - 3u; + add_node_index = model->info.node_count - 2u; + matmul_node = model->bytes + (size_t)model->node_offset + + (size_t)matmul_node_index * LWM_V0_NODE_SIZE; + add_node = model->bytes + (size_t)model->node_offset + + (size_t)add_node_index * LWM_V0_NODE_SIZE; + if (lwm_read_u16(matmul_node) != LW_OP_MATMUL || + lwm_read_u16(matmul_node + 2u) != 2u || lwm_read_u16(matmul_node + 4u) != 1u || + lwm_read_u16(add_node) != LW_OP_ADD || lwm_read_u16(add_node + 2u) != 2u || + lwm_read_u16(add_node + 4u) != 1u || lwm_read_u32(add_node + 40u) != logits_index) { + return 0; + } + matmul_input_index = lwm_read_u32(matmul_node + 8u); + weights_index = lwm_read_u32(matmul_node + 12u); + matmul_output_index = lwm_read_u32(matmul_node + 40u); + if (lwm_read_u32(add_node + 8u) == matmul_output_index) { + bias_index = lwm_read_u32(add_node + 12u); + } else if (lwm_read_u32(add_node + 12u) == matmul_output_index) { + bias_index = lwm_read_u32(add_node + 8u); + } else { + return 0; + } + activation = &session->tensors[matmul_input_index]; + weights = &session->tensors[weights_index]; + matmul_output = &session->tensors[matmul_output_index]; + bias = &session->tensors[bias_index]; + prepared = prepared_constant_for_node(session, matmul_node_index, profile); + if (prepared == NULL || prepared->kind != LW_PREPARED_CONSTANT_MATMUL_PACKED16 || + activation->dtype != LW_DTYPE_F32 || activation->rank != 3u || + activation->dimensions[0] != 1 || activation->dimensions[1] != (int32_t)time_steps || + activation->dimensions[2] <= 0 || weights->dtype != LW_DTYPE_F32 || + weights->rank != 2u || weights->dimensions[0] != activation->dimensions[2] || + weights->dimensions[1] != (int32_t)class_count || + (weights->flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u || + matmul_output->dtype != LW_DTYPE_F32 || matmul_output->rank != 3u || + matmul_output->dimensions[0] != 1 || + matmul_output->dimensions[1] != (int32_t)time_steps || + matmul_output->dimensions[2] != (int32_t)class_count || bias->dtype != LW_DTYPE_F32 || + bias->rank != 1u || bias->dimensions[0] != (int32_t)class_count || + (bias->flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u) { + return 0; + } + graph_input_index = lwm_read_u32(model->bytes + (size_t)model->input_offset); + projection->matmul_node_index = matmul_node_index; + projection->add_node_index = add_node_index; + projection->activation = tensor_input_data(session, matmul_input_index, graph_input_index, + graph_input); + projection->packed_weights = + (const float*)(const void*)(session->packed_weights + + (size_t)prepared->packed_weight_offset); + projection->bias = tensor_input_data(session, bias_index, graph_input_index, graph_input); + projection->logits = tensor_output_data(session, matmul_output_index); + projection->rows = time_steps; + projection->inner_dimension = (uint32_t)activation->dimensions[2]; + projection->columns = class_count; + return projection->activation != NULL && projection->bias != NULL && + projection->logits != NULL; +} + +static void profile_simple_node(lw_execution_profile* profile, const lw_session* session, + uint32_t node_index, uint64_t elapsed) { + const uint8_t* node; + uint32_t operation; + if (profile == NULL || node_index >= session->model->info.node_count) { + return; + } + if (elapsed == 0u) { + elapsed = 1u; + } + node = session->model->bytes + (size_t)session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + operation = (uint32_t)lwm_read_u16(node); + if (operation < LW_EXECUTION_PROFILE_OPERATOR_CAPACITY && + profile->operator_nanoseconds[operation] <= UINT64_MAX - elapsed && + profile->operator_invocations[operation] != UINT64_MAX) { + profile->operator_nanoseconds[operation] += elapsed; + ++profile->operator_invocations[operation]; + } + if (node_index < LW_EXECUTION_PROFILE_NODE_CAPACITY && + profile->node_nanoseconds[node_index] <= UINT64_MAX - elapsed && + profile->node_invocations[node_index] != UINT64_MAX) { + profile->node_nanoseconds[node_index] += elapsed; + ++profile->node_invocations[node_index]; + } +} + +#if defined(LW_EXPERIMENTAL_CTC_TILED) +static lw_status execute_packed_ctc_projection_tiled( + lw_session* session, const lw_packed_ctc_projection* projection, uint32_t* best_indices, + float* best_probabilities, uint32_t time_steps, uint32_t class_count, + uint64_t* projection_elapsed, uint64_t* softmax_elapsed, lw_execution_profile* profile, + lw_error* error) { + uint32_t offset = 0u; + uint64_t projection_started = profile == NULL ? 0u : profile->clock(profile->clock_context); + uint64_t softmax_total = 0u; + uint64_t projection_total = 0u; + while (offset < time_steps) { + uint32_t rows = time_steps - offset; + uint64_t activation_offset; + uint64_t scratch_elements; + uint64_t block_started = profile == NULL ? 0u : profile->clock(profile->clock_context); + uint64_t block_finished; + uint64_t block_elapsed = 0u; + uint64_t softmax_started; + uint64_t softmax_finished; + uint64_t softmax_block_elapsed = 0u; + lw_status status; + if (rows > session->ctc_logits_scratch_rows) { + rows = session->ctc_logits_scratch_rows; + } + if (rows == 0u || offset > UINT32_MAX - rows || + (uint64_t)offset * projection->inner_dimension > UINT64_MAX) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "CTC tiled projection dimensions overflow"); + return LW_STATUS_OUT_OF_BOUNDS; + } + activation_offset = (uint64_t)offset * projection->inner_dimension; + scratch_elements = (uint64_t)rows * class_count; + if (scratch_elements > session->ctc_logits_scratch_elements || + activation_offset > (uint64_t)SIZE_MAX || + (uint64_t)(size_t)activation_offset != activation_offset) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "CTC tiled projection buffer is too large"); + return LW_STATUS_OUT_OF_BOUNDS; + } + { + uint32_t head_workers = 1u; + lw_thread_pool* head_pool = lw_session_ctc_head_pool(session, &head_workers); + head_workers = lw_ctc_head_parallel_worker_count(head_pool, head_workers, rows, + projection->inner_dimension, + projection->columns); + if (head_workers > 1u) { + /* Row-parallel head projection: bit-identical per element. */ + lw_ctc_head_bias_argmax_parallel_f32( + head_pool, head_workers, + projection->activation + (size_t)activation_offset, + projection->packed_weights, projection->bias, session->ctc_logits_scratch, + best_indices + offset, rows, projection->inner_dimension, + projection->columns); + } else { + lw_avx2_packed_matmul_bias_argmax_f32( + projection->activation + (size_t)activation_offset, projection->packed_weights, + projection->bias, session->ctc_logits_scratch, best_indices + offset, 1u, rows, + projection->inner_dimension, projection->columns); + } + } + block_finished = profile == NULL ? 0u : profile->clock(profile->clock_context); + if (profile != NULL && block_finished >= block_started) { + block_elapsed = block_finished - block_started; + } + projection_total += block_elapsed; + softmax_started = profile == NULL ? 0u : profile->clock(profile->clock_context); + status = lw_ctc_emitted_softmax_contiguous_f32( + session->ctc_logits_scratch, best_indices + offset, + best_probabilities + offset, rows, class_count); + softmax_finished = profile == NULL ? 0u : profile->clock(profile->clock_context); + if (profile != NULL && softmax_finished >= softmax_started) { + softmax_block_elapsed = softmax_finished - softmax_started; + } + softmax_total += softmax_block_elapsed; + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "tiled CTC logits contain invalid values"); + return status; + } + if (offset != 0u && best_indices[offset] != 0u && + best_indices[offset] == best_indices[offset - 1u]) { + best_probabilities[offset] = 0.0f; + } + offset += rows; + } + if (profile != NULL) { + uint64_t projection_finished = profile->clock(profile->clock_context); + if (projection_finished >= projection_started && projection_total == 0u) { + uint64_t total_elapsed = projection_finished - projection_started; + if (total_elapsed >= softmax_total) { + projection_total = total_elapsed - softmax_total; + } + } + } + if (projection_elapsed != NULL) { + *projection_elapsed = projection_total; + } + if (softmax_elapsed != NULL) { + *softmax_elapsed = softmax_total; + } + return LW_STATUS_OK; +} +#endif + +lw_status lw_execute_session_f32_ctc_greedy( + lw_session* session, const float* input, uint64_t input_element_count, + uint32_t* best_indices, float* best_probabilities, uint32_t time_steps, + uint32_t class_count, lw_execution_profile* profile, lw_error* error) { + uint32_t logits_index; + uint32_t softmax_node_index; + lw_packed_ctc_projection projection; + const float* logits; + int fused_projection; + int tiled_projection = 0; + int softmax_done = 0; + uint64_t started = 0u; + uint64_t elapsed = 0u; + lw_status status; + if (session == NULL || input == NULL || best_indices == NULL || best_probabilities == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "session, input, and CTC greedy outputs are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (profile != NULL && + (profile->struct_size != sizeof(*profile) || profile->reserved != 0u || + profile->clock == NULL)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "an initialized execution profile and clock are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (!ctc_greedy_tail(session, time_steps, class_count, &logits_index)) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "session output is not a supported terminal CTC Softmax"); + return LW_STATUS_UNSUPPORTED; + } + softmax_node_index = session->model->info.node_count - 1u; + fused_projection = match_packed_ctc_projection(session, input, logits_index, time_steps, + class_count, profile, &projection); + if (profile != NULL) { + if (profile->ctc_greedy_invocations != UINT64_MAX) { + ++profile->ctc_greedy_invocations; + } + if (fused_projection) { + if (profile->ctc_packed_projection_invocations != UINT64_MAX) { + ++profile->ctc_packed_projection_invocations; + } + if (profile->packed_matmul_invocations != UINT64_MAX) { + ++profile->packed_matmul_invocations; + } + } else if (profile->ctc_generic_projection_invocations != UINT64_MAX) { + ++profile->ctc_generic_projection_invocations; + } + } + status = execute_session_nodes_f32( + session, input, input_element_count, + fused_projection ? projection.matmul_node_index : softmax_node_index, profile, error); + if (status != LW_STATUS_OK) { + return status; + } + if (fused_projection) { + uint64_t projection_started = profile == NULL ? 0u : profile->clock(profile->clock_context); + uint64_t projection_elapsed = 0u; +#if defined(LW_EXPERIMENTAL_CTC_TILED) + if (session->ctc_logits_scratch != NULL && session->ctc_logits_scratch_rows != 0u) { + uint64_t tiled_softmax_elapsed = 0u; + status = execute_packed_ctc_projection_tiled( + session, &projection, best_indices, best_probabilities, time_steps, class_count, + &projection_elapsed, &tiled_softmax_elapsed, profile, error); + if (status != LW_STATUS_OK) { + return status; + } + tiled_projection = 1; + softmax_done = 1; + if (profile != NULL) { + profile_simple_node(profile, session, projection.matmul_node_index, + projection_elapsed); + profile_simple_node(profile, session, projection.add_node_index, 1u); + profile_simple_node(profile, session, softmax_node_index, + tiled_softmax_elapsed); + } + } else +#endif + { + uint32_t head_workers = 1u; + lw_thread_pool* head_pool = lw_session_ctc_head_pool(session, &head_workers); + head_workers = lw_ctc_head_parallel_worker_count(head_pool, head_workers, + projection.rows, + projection.inner_dimension, + projection.columns); + if (head_workers > 1u) { + /* Row-parallel head projection: bit-identical per element. */ + lw_ctc_head_bias_argmax_parallel_f32( + head_pool, head_workers, projection.activation, projection.packed_weights, + projection.bias, projection.logits, best_indices, projection.rows, + projection.inner_dimension, projection.columns); + } else { + lw_avx2_packed_matmul_bias_argmax_f32( + projection.activation, projection.packed_weights, projection.bias, + projection.logits, best_indices, 1u, projection.rows, + projection.inner_dimension, projection.columns); + } + } + if (profile != NULL) { + if (!tiled_projection) { + uint64_t projection_finished = profile->clock(profile->clock_context); + if (projection_finished >= projection_started) { + projection_elapsed = projection_finished - projection_started; + } + profile_simple_node(profile, session, projection.matmul_node_index, + projection_elapsed); + profile_simple_node(profile, session, projection.add_node_index, 1u); + } + } + logits = projection.logits; + } else { + logits = tensor_input_data( + session, logits_index, + lwm_read_u32(session->model->bytes + (size_t)session->model->input_offset), input); + } + if (!softmax_done && profile != NULL) { + started = profile->clock(profile->clock_context); + } + if (!softmax_done) { + status = fused_projection + ? lw_ctc_emitted_softmax_contiguous_f32( + logits, best_indices, best_probabilities, time_steps, class_count) + : lw_ctc_greedy_softmax_contiguous_f32( + logits, best_indices, best_probabilities, time_steps, class_count); + } + if (!softmax_done && profile != NULL) { + uint64_t finished = profile->clock(profile->clock_context); + if (finished >= started) { + elapsed = finished - started; + } + if (elapsed == 0u) { + elapsed = 1u; + } + if (profile->operator_nanoseconds[LW_OP_SOFTMAX] <= UINT64_MAX - elapsed && + profile->operator_invocations[LW_OP_SOFTMAX] != UINT64_MAX) { + profile->operator_nanoseconds[LW_OP_SOFTMAX] += elapsed; + ++profile->operator_invocations[LW_OP_SOFTMAX]; + } + if (softmax_node_index < LW_EXECUTION_PROFILE_NODE_CAPACITY && + profile->node_nanoseconds[softmax_node_index] <= UINT64_MAX - elapsed && + profile->node_invocations[softmax_node_index] != UINT64_MAX) { + profile->node_nanoseconds[softmax_node_index] += elapsed; + ++profile->node_invocations[softmax_node_index]; + } + } + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "terminal CTC logits contain invalid values"); + return status; + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +lw_status lw_execute_session_f32(lw_session* session, const float* input, + uint64_t input_element_count, float* output, + uint64_t output_element_count, lw_error* error) { + return execute_session_f32(session, input, input_element_count, output, output_element_count, + NULL, error); +} + +lw_status lw_execute_session_f32_profiled(lw_session* session, const float* input, + uint64_t input_element_count, float* output, + uint64_t output_element_count, + lw_execution_profile* profile, lw_error* error) { + if (profile == NULL || profile->struct_size != sizeof(*profile) || profile->reserved != 0u || + profile->clock == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "an initialized execution profile and clock are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + return execute_session_f32(session, input, input_element_count, output, output_element_count, + profile, error); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/executor_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/executor_internal.h new file mode 100644 index 00000000..9499d1f5 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/executor_internal.h @@ -0,0 +1,101 @@ +#ifndef LW_EXECUTOR_INTERNAL_H +#define LW_EXECUTOR_INTERNAL_H + +#include "lw_infer.h" + +#include + +#define LW_EXECUTION_PROFILE_OPERATOR_CAPACITY 22u +#define LW_EXECUTION_PROFILE_NODE_CAPACITY 256u +#define LW_EXECUTION_PROFILE_CONV_CLASS_CAPACITY 5u +#define LW_EXECUTION_PROFILE_THREAD_HISTOGRAM_CAPACITY 17u + +typedef enum lw_execution_profile_conv_class { + LW_EXECUTION_PROFILE_CONV_1X1 = 0, + LW_EXECUTION_PROFILE_CONV_3X3 = 1, + LW_EXECUTION_PROFILE_CONV_DEPTHWISE_3X3 = 2, + LW_EXECUTION_PROFILE_CONV_STRIDE2_3X3 = 3, + LW_EXECUTION_PROFILE_CONV_OTHER = 4 +} lw_execution_profile_conv_class; + +typedef uint64_t (*lw_execution_profile_clock)(void* context); + +typedef struct lw_execution_profile { + uint32_t struct_size; + uint32_t reserved; + lw_execution_profile_clock clock; + void* clock_context; + uint64_t operator_nanoseconds[LW_EXECUTION_PROFILE_OPERATOR_CAPACITY]; + uint64_t operator_invocations[LW_EXECUTION_PROFILE_OPERATOR_CAPACITY]; + uint64_t node_nanoseconds[LW_EXECUTION_PROFILE_NODE_CAPACITY]; + uint64_t node_invocations[LW_EXECUTION_PROFILE_NODE_CAPACITY]; + uint64_t conv_class_nanoseconds[LW_EXECUTION_PROFILE_CONV_CLASS_CAPACITY]; + uint64_t conv_class_invocations[LW_EXECUTION_PROFILE_CONV_CLASS_CAPACITY]; + /* Index is the actual thread count selected for one Conv invocation. + * Index zero remains unused so reports can print the policy directly. */ + uint64_t conv_thread_histogram[LW_EXECUTION_PROFILE_THREAD_HISTOGRAM_CAPACITY]; + /* ConvTranspose is tracked separately so the established Conv histogram + * keeps its original meaning. */ + uint64_t conv_transpose_thread_histogram[LW_EXECUTION_PROFILE_THREAD_HISTOGRAM_CAPACITY]; + /* Implementation-path counters are intentionally internal. They make + * hotspot reports actionable without changing the public C ABI. */ + uint64_t prepared_binding_lookups; + uint64_t prepared_binding_hits; + uint64_t prepared_binding_fallbacks; + uint64_t prepared_node_invocations; + uint64_t generic_node_invocations; + uint64_t prepared_conv1x1_invocations; + uint64_t prepared_conv3x3_invocations; + uint64_t packed_conv1x1_invocations; + uint64_t packed_conv3x3_stride2_invocations; + uint64_t unpacked_conv_invocations; + uint64_t packed_matmul_invocations; + uint64_t unpacked_matmul_invocations; + uint64_t fused_gelu_invocations; + uint64_t ctc_greedy_invocations; + uint64_t ctc_packed_projection_invocations; + uint64_t ctc_generic_projection_invocations; + /* Layout telemetry is internal and does not change the public C ABI. */ + uint64_t layout_transform_nanoseconds; + uint64_t layout_transform_invocations; + uint64_t layout_transform_bytes; + uint64_t layout_candidate_nodes; + uint64_t layout_selected_nodes; + uint64_t layout_fallback_nodes; + uint64_t layout_analysis_runs; +} lw_execution_profile; + +const float* lw_executor_tensor_input_data(const lw_session* session, + uint32_t tensor_index, + uint32_t graph_input_index, + const float* graph_input); +float* lw_executor_tensor_output_data(lw_session* session, uint32_t tensor_index); +lw_status lw_executor_dispatch_node_f32(lw_session* session, + uint32_t node_index, + uint32_t graph_input_index, + const float* graph_input, + lw_execution_profile* profile); +lw_status lw_executor_execute_best_node_f32(lw_session* session, + uint32_t node_index, + uint32_t graph_input_index, + const float* graph_input, + lw_execution_profile* profile, + uint32_t* consumed_nodes); + +lw_status lw_execute_session_f32(lw_session* session, const float* input, + uint64_t input_element_count, float* output, + uint64_t output_element_count, lw_error* error); + +lw_status lw_execute_session_f32_profiled(lw_session* session, const float* input, + uint64_t input_element_count, float* output, + uint64_t output_element_count, + lw_execution_profile* profile, lw_error* error); + +int lw_session_supports_ctc_greedy_f32(const lw_session* session, uint32_t time_steps, + uint32_t class_count); +lw_status lw_execute_session_f32_ctc_greedy( + lw_session* session, const float* input, uint64_t input_element_count, + uint32_t* best_indices, float* best_probabilities, uint32_t time_steps, + uint32_t class_count, lw_execution_profile* profile, lw_error* error); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/layout_convert_x64.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/layout_convert_x64.c new file mode 100644 index 00000000..af7d0c54 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/layout_convert_x64.c @@ -0,0 +1,191 @@ +#include "x64_rec_fast_internal.h" + +#include + +#if defined(_M_X64) || defined(__x86_64__) || defined(_M_IX86) || defined(__i386__) +#include +#define LW_CONVERT_X86 1 +#else +#define LW_CONVERT_X86 0 +#endif + +#if LW_CONVERT_X86 && (defined(__GNUC__) || defined(__clang__)) +#define LW_CONVERT_TARGET __attribute__((target("avx2,fma"))) +#else +#define LW_CONVERT_TARGET +#endif + +#if LW_CONVERT_X86 +/* Transpose an 8x8 block of floats: v0..v7 hold 8-pixel rows of 8 channels + * each; t0..t7 receive 8-channel rows of 8 pixels each. */ +LW_CONVERT_TARGET +static void transpose8x8_ps(const __m256* v, __m256* t) { + /* Classic AVX2 8x8 float transpose: unpack adjacent row pairs, shuffle + * quads, then permute 128-bit lanes. */ + __m256 a[8]; + __m256 b[8]; + a[0] = _mm256_unpacklo_ps(v[0], v[1]); + a[1] = _mm256_unpackhi_ps(v[0], v[1]); + a[2] = _mm256_unpacklo_ps(v[2], v[3]); + a[3] = _mm256_unpackhi_ps(v[2], v[3]); + a[4] = _mm256_unpacklo_ps(v[4], v[5]); + a[5] = _mm256_unpackhi_ps(v[4], v[5]); + a[6] = _mm256_unpacklo_ps(v[6], v[7]); + a[7] = _mm256_unpackhi_ps(v[6], v[7]); + b[0] = _mm256_shuffle_ps(a[0], a[2], 0x44); + b[1] = _mm256_shuffle_ps(a[0], a[2], 0xEE); + b[2] = _mm256_shuffle_ps(a[1], a[3], 0x44); + b[3] = _mm256_shuffle_ps(a[1], a[3], 0xEE); + b[4] = _mm256_shuffle_ps(a[4], a[6], 0x44); + b[5] = _mm256_shuffle_ps(a[4], a[6], 0xEE); + b[6] = _mm256_shuffle_ps(a[5], a[7], 0x44); + b[7] = _mm256_shuffle_ps(a[5], a[7], 0xEE); + t[0] = _mm256_permute2f128_ps(b[0], b[4], 0x20); + t[1] = _mm256_permute2f128_ps(b[1], b[5], 0x20); + t[2] = _mm256_permute2f128_ps(b[2], b[6], 0x20); + t[3] = _mm256_permute2f128_ps(b[3], b[7], 0x20); + t[4] = _mm256_permute2f128_ps(b[0], b[4], 0x31); + t[5] = _mm256_permute2f128_ps(b[1], b[5], 0x31); + t[6] = _mm256_permute2f128_ps(b[2], b[6], 0x31); + t[7] = _mm256_permute2f128_ps(b[3], b[7], 0x31); +} +#endif + +LW_CONVERT_TARGET +void lw_x64_fast_nchw_to_nhwc(const float* source, float* destination, + uint32_t batch, uint32_t channels, + uint32_t height, uint32_t width) { +#if LW_CONVERT_X86 + /* Vectorized 8x8 path for 8-multiple channels; the scalar fallback + * covers the channel/width tails and odd channel counts. */ + if (channels % 8u == 0u) { + uint32_t n; + for (n = 0u; n < batch; ++n) { + uint32_t y; + for (y = 0u; y < height; ++y) { + uint32_t x = 0u; + while (x + 8u <= width) { + uint32_t c = 0u; + while (c + 8u <= channels) { + __m256 v[8]; + __m256 t[8]; + uint32_t k; + for (k = 0u; k < 8u; ++k) { + v[k] = _mm256_loadu_ps( + source + ((size_t)n * channels + c + k) * height * width + + (size_t)y * width + x); + } + transpose8x8_ps(v, t); + for (k = 0u; k < 8u; ++k) { + _mm256_storeu_ps( + destination + (((size_t)n * height + y) * width + x + k) * + channels + c, t[k]); + } + c += 8u; + } + while (c < channels) { + uint32_t k; + for (k = 0u; k < 8u; ++k) { + destination[(((size_t)n * height + y) * width + x + k) * + channels + c] = + source[((size_t)n * channels + c) * height * width + + (size_t)y * width + x + k]; + } + ++c; + } + x += 8u; + } + while (x < width) { + uint32_t c; + for (c = 0u; c < channels; ++c) { + destination[(((size_t)n * height + y) * width + x) * channels + c] = + source[((size_t)n * channels + c) * height * width + + (size_t)y * width + x]; + } + ++x; + } + } + } + return; + } +#endif + for (uint32_t n = 0u; n < batch; ++n) { + for (uint32_t y = 0u; y < height; ++y) { + for (uint32_t x = 0u; x < width; ++x) { + float* dst = destination + (((size_t)n * height + y) * width + x) * channels; + for (uint32_t c = 0u; c < channels; ++c) { + dst[c] = source[(((size_t)n * channels + c) * height + y) * width + x]; + } + } + } + } +} + +LW_CONVERT_TARGET +void lw_x64_fast_nhwc_to_nchw(const float* source, float* destination, + uint32_t batch, uint32_t channels, + uint32_t height, uint32_t width) { +#if LW_CONVERT_X86 + if (channels % 8u == 0u) { + uint32_t n; + for (n = 0u; n < batch; ++n) { + uint32_t y; + for (y = 0u; y < height; ++y) { + uint32_t x = 0u; + while (x + 8u <= width) { + uint32_t c = 0u; + while (c + 8u <= channels) { + __m256 v[8]; + __m256 t[8]; + uint32_t k; + for (k = 0u; k < 8u; ++k) { + v[k] = _mm256_loadu_ps( + source + (((size_t)n * height + y) * width + x + k) * + channels + c); + } + transpose8x8_ps(v, t); + for (k = 0u; k < 8u; ++k) { + _mm256_storeu_ps( + destination + ((size_t)n * channels + c + k) * height * width + + (size_t)y * width + x, t[k]); + } + c += 8u; + } + while (c < channels) { + uint32_t k; + for (k = 0u; k < 8u; ++k) { + destination[((size_t)n * channels + c) * height * width + + (size_t)y * width + x + k] = + source[(((size_t)n * height + y) * width + x + k) * + channels + c]; + } + ++c; + } + x += 8u; + } + while (x < width) { + uint32_t c; + for (c = 0u; c < channels; ++c) { + destination[((size_t)n * channels + c) * height * width + + (size_t)y * width + x] = + source[(((size_t)n * height + y) * width + x) * channels + c]; + } + ++x; + } + } + } + return; + } +#endif + for (uint32_t n = 0u; n < batch; ++n) { + for (uint32_t c = 0u; c < channels; ++c) { + float* dst = destination + ((size_t)n * channels + c) * height * width; + for (uint32_t y = 0u; y < height; ++y) { + for (uint32_t x = 0u; x < width; ++x) { + dst[(size_t)y * width + x] = + source[(((size_t)n * height + y) * width + x) * channels + c]; + } + } + } + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/layout_planner.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/layout_planner.c new file mode 100644 index 00000000..026bd3e5 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/layout_planner.c @@ -0,0 +1,638 @@ +#include "layout_planner_internal.h" + +#include "lwm_read.h" +#include "model_internal.h" +#include "operator_internal.h" +#include "session_internal.h" + +#include +#include +#include +#include + +static const uint8_t* layout_node_bytes(const lw_session* session, uint32_t node_index) { + if (session == NULL || session->model == NULL || node_index >= session->model->info.node_count) { + return NULL; + } + return session->model->bytes + (size_t)session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; +} + +static const uint8_t* layout_node_params(const lw_session* session, const uint8_t* node) { + uint64_t offset; + if (session == NULL || session->model == NULL || node == NULL) { + return NULL; + } + offset = lwm_read_u64(node + 56u); + if (offset == 0u || offset > session->model->byte_count || + session->model->byte_count - (size_t)offset < 4u) { + return NULL; + } + return session->model->bytes + (size_t)offset; +} + +static float layout_read_f32(const uint8_t* bytes) { + float value = 0.0f; + if (bytes != NULL) { + memcpy(&value, bytes, sizeof(value)); + } + return value; +} +static int layout_positive_integer_scale(float value) { + uint32_t integer; + if (!(value >= 1.0f) || value > 16777216.0f) { + return 0; + } + integer = (uint32_t)value; + return (float)integer == value; +} +static int layout_tensor_valid(const lw_session* session, uint32_t tensor_index) { + return session != NULL && session->model != NULL && session->tensors != NULL && + tensor_index < session->model->info.tensor_count; +} + +static int layout_tensor_is_constant(const lw_session* session, uint32_t tensor_index) { + return layout_tensor_valid(session, tensor_index) && + (session->tensors[tensor_index].flags & LWM_V0_TENSOR_FLAG_CONSTANT) != 0u; +} + +static uint64_t layout_tensor_elements(const lw_runtime_tensor* tensor) { + if (tensor == NULL || tensor->dtype != LW_DTYPE_F32 || tensor->byte_size % sizeof(float) != 0u) { + return 0u; + } + return tensor->byte_size / sizeof(float); +} + +static int layout_same_shape(const lw_runtime_tensor* left, const lw_runtime_tensor* right) { + uint32_t dimension; + if (left == NULL || right == NULL || left->rank != right->rank) { + return 0; + } + for (dimension = 0u; dimension < left->rank && dimension < LW_MAX_DIMS; ++dimension) { + if (left->dimensions[dimension] != right->dimensions[dimension]) { + return 0; + } + } + return 1; +} + +static int layout_rank4_f32(const lw_runtime_tensor* tensor) { + return tensor != NULL && tensor->dtype == LW_DTYPE_F32 && tensor->rank == 4u && + tensor->dimensions[0] > 0 && tensor->dimensions[1] > 0 && + tensor->dimensions[2] > 0 && tensor->dimensions[3] > 0; +} + +int lw_layout_tensor_is_neutral(const lw_session* session, uint32_t tensor_index) { + const lw_runtime_tensor* tensor; + if (!layout_tensor_valid(session, tensor_index)) { + return 0; + } + tensor = &session->tensors[tensor_index]; + return layout_rank4_f32(tensor) && + (tensor->dimensions[1] == 1 || + (tensor->dimensions[2] == 1 && tensor->dimensions[3] == 1)); +} + +static int layout_channel_vector(const lw_runtime_tensor* tensor, + const lw_runtime_tensor* full) { + if (!layout_rank4_f32(tensor) || !layout_rank4_f32(full)) { + return 0; + } + return tensor->dimensions[1] == full->dimensions[1] && tensor->dimensions[2] == 1 && + tensor->dimensions[3] == 1 && + (tensor->dimensions[0] == 1 || tensor->dimensions[0] == full->dimensions[0]); +} + +static int layout_channel_broadcast(const lw_session* session, uint32_t tensor_index, + const lw_runtime_tensor* full) { + const lw_runtime_tensor* tensor; + if (!layout_tensor_valid(session, tensor_index) || full == NULL) { + return 0; + } + tensor = &session->tensors[tensor_index]; + if (layout_tensor_elements(tensor) == 1u) { + return 1; + } + if (layout_tensor_is_constant(session, tensor_index) && + layout_tensor_elements(tensor) == (uint64_t)(uint32_t)full->dimensions[1]) { + return 1; + } + return layout_channel_vector(tensor, full); +} +static int layout_binary_compatible(const lw_session* session, uint32_t left_index, + uint32_t right_index, uint32_t output_index) { + const lw_runtime_tensor* left; + const lw_runtime_tensor* right; + const lw_runtime_tensor* output; + if (!layout_tensor_valid(session, left_index) || !layout_tensor_valid(session, right_index) || + !layout_tensor_valid(session, output_index)) { + return 0; + } + left = &session->tensors[left_index]; + right = &session->tensors[right_index]; + output = &session->tensors[output_index]; + if (!layout_rank4_f32(output)) { + return 0; + } + if (layout_channel_broadcast(session, left_index, output) || + layout_channel_broadcast(session, right_index, output)) { + return 1; + } + return layout_same_shape(left, right) && layout_same_shape(left, output); +} + +static int layout_conv_capable(const lw_session* session, const uint8_t* node, + const uint8_t* params) { + uint32_t input_count; + uint32_t input_index; + uint32_t weight_index; + uint32_t group; + int32_t kernel_height; + int32_t kernel_width; + int32_t stride_height; + int32_t stride_width; + int32_t dilation_height; + int32_t dilation_width; + const lw_runtime_tensor* input; + const lw_runtime_tensor* weight; + const lw_runtime_tensor* output; + if (node == NULL || params == NULL || lwm_read_u16(node + 4u) != 1u) { + return 0; + } + input_count = lwm_read_u16(node + 2u); + if (input_count < 2u || input_count > 3u) { + return 0; + } + input_index = lwm_read_u32(node + 8u); + weight_index = lwm_read_u32(node + 12u); + if (!layout_tensor_valid(session, input_index) || !layout_tensor_valid(session, weight_index) || + !layout_tensor_valid(session, lwm_read_u32(node + 40u)) || + !layout_tensor_is_constant(session, weight_index)) { + return 0; + } + input = &session->tensors[input_index]; + weight = &session->tensors[weight_index]; + output = &session->tensors[lwm_read_u32(node + 40u)]; + if (!layout_rank4_f32(input) || !layout_rank4_f32(output) || weight->rank != 4u || + weight->dtype != LW_DTYPE_F32) { + return 0; + } + group = lwm_read_u32(params + 4u); + kernel_height = lwm_read_i32(params + 8u); + kernel_width = lwm_read_i32(params + 12u); + stride_height = lwm_read_i32(params + 16u); + stride_width = lwm_read_i32(params + 20u); + dilation_height = lwm_read_i32(params + 24u); + dilation_width = lwm_read_i32(params + 28u); + if (group == 0u || kernel_height <= 0 || kernel_width <= 0 || stride_height <= 0 || + stride_width <= 0 || dilation_height != 1 || dilation_width != 1 || + weight->dimensions[2] != kernel_height || weight->dimensions[3] != kernel_width) { + return 0; + } + if (group == 1u) { + /* The NHWC kernels pack in 16-lane blocks and store partial tail + * blocks lane-exactly, so any channel count >= 8 qualifies (the + * 24-channel small-model family, and the 12-channel head conv, would + * otherwise fall back to the serial NCHW arm). */ + return weight->dimensions[0] >= 8; + } + return group == (uint32_t)input->dimensions[1] && weight->dimensions[1] == 1 && + (group & 7u) == 0u; +} + +static int layout_conv_transpose_capable(const lw_session* session, const uint8_t* node, + const uint8_t* params) { + uint32_t input_index; + uint32_t weight_index; + const lw_runtime_tensor* input; + const lw_runtime_tensor* weight; + const lw_runtime_tensor* output; + if (node == NULL || params == NULL || lwm_read_u16(node + 2u) < 2u || + lwm_read_u16(node + 4u) != 1u) { + return 0; + } + input_index = lwm_read_u32(node + 8u); + weight_index = lwm_read_u32(node + 12u); + if (!layout_tensor_valid(session, input_index) || !layout_tensor_valid(session, weight_index) || + !layout_tensor_valid(session, lwm_read_u32(node + 40u)) || + !layout_tensor_is_constant(session, weight_index)) { + return 0; + } + input = &session->tensors[input_index]; + weight = &session->tensors[weight_index]; + output = &session->tensors[lwm_read_u32(node + 40u)]; + return layout_rank4_f32(input) && layout_rank4_f32(output) && weight->rank == 4u && + weight->dtype == LW_DTYPE_F32 && lwm_read_u32(params + 4u) == 1u && + lwm_read_i32(params + 8u) == 2 && lwm_read_i32(params + 12u) == 2 && + lwm_read_i32(params + 16u) == 2 && lwm_read_i32(params + 20u) == 2 && + lwm_read_i32(params + 24u) == 1 && lwm_read_i32(params + 28u) == 1 && + lwm_read_i32(params + 32u) == 0 && lwm_read_i32(params + 36u) == 0 && + lwm_read_i32(params + 40u) == 0 && lwm_read_i32(params + 44u) == 0; +} + +static int layout_elementwise_capable(const lw_session* session, const uint8_t* node, + uint16_t operation) { + uint32_t input_count = lwm_read_u16(node + 2u); + uint32_t output_index = lwm_read_u32(node + 40u); + uint32_t input_index; + const lw_runtime_tensor* input; + const lw_runtime_tensor* output; + if (input_count == 0u || lwm_read_u16(node + 4u) != 1u || + !layout_tensor_valid(session, output_index)) { + return 0; + } + input_index = lwm_read_u32(node + 8u); + if (!layout_tensor_valid(session, input_index)) { + return 0; + } + input = &session->tensors[input_index]; + output = &session->tensors[output_index]; + if (!layout_rank4_f32(input) || !layout_rank4_f32(output) || + !layout_same_shape(input, output)) { + return 0; + } + if (operation == LW_OP_ADD || operation == LW_OP_MUL || operation == LW_OP_DIV || + operation == LW_OP_SUB || operation == LW_OP_POW) { + return input_count == 2u && + layout_binary_compatible(session, input_index, lwm_read_u32(node + 12u), + output_index); + } + return operation == LW_OP_RELU || operation == LW_OP_SIGMOID || operation == LW_OP_SQRT || + operation == LW_OP_ERF || operation == LW_OP_HARD_SIGMOID; +} + +static int layout_reduce_mean_capable(const lw_session* session, const uint8_t* node, + const uint8_t* params) { + uint32_t input_index; + uint32_t output_index; + int32_t axis0; + int32_t axis1; + const lw_runtime_tensor* input; + const lw_runtime_tensor* output; + if (node == NULL || params == NULL || lwm_read_u16(node + 2u) != 1u || + lwm_read_u16(node + 4u) != 1u) { + return 0; + } + input_index = lwm_read_u32(node + 8u); + output_index = lwm_read_u32(node + 40u); + if (!layout_tensor_valid(session, input_index) || + !layout_tensor_valid(session, output_index)) { + return 0; + } + input = &session->tensors[input_index]; + output = &session->tensors[output_index]; + if (!layout_rank4_f32(input) || !layout_rank4_f32(output) || + lwm_read_u16(params + 2u) != 2u || lwm_read_u32(params + 4u) == 0u) { + return 0; + } + axis0 = lwm_read_i32(params + 12u); + axis1 = lwm_read_i32(params + 16u); + if (axis0 < 0) { + axis0 += 4; + } + if (axis1 < 0) { + axis1 += 4; + } + return (axis0 == 2 && axis1 == 3) || (axis0 == 3 && axis1 == 2); +} + +static int layout_concat_capable(const lw_session* session, const uint8_t* node, + const uint8_t* params) { + uint32_t input_count; + uint32_t input_slot; + int32_t axis; + const lw_runtime_tensor* output; + if (node == NULL || params == NULL || + (input_count = lwm_read_u16(node + 2u)) == 0u || + lwm_read_u16(node + 4u) != 1u) { + return 0; + } + if (!layout_tensor_valid(session, lwm_read_u32(node + 40u))) { + return 0; + } + output = &session->tensors[lwm_read_u32(node + 40u)]; + if (!layout_rank4_f32(output)) { + return 0; + } + axis = lwm_read_i32(params + 4u); + if (axis < 0) { + axis += 4; + } + if (axis != 1) { + return 0; + } + for (input_slot = 0u; input_slot < input_count && input_slot < LWM_V0_MAX_NODE_INPUTS; + ++input_slot) { + uint32_t input_index = lwm_read_u32( + node + 8u + (size_t)input_slot * sizeof(uint32_t)); + if (!layout_tensor_valid(session, input_index) || + layout_tensor_is_constant(session, input_index) || + !layout_rank4_f32(&session->tensors[input_index])) { + return 0; + } + } + return input_count <= LWM_V0_MAX_NODE_INPUTS; +} +int lw_layout_node_nhwc_capable(const lw_session* session, uint32_t node_index) { + const uint8_t* node = layout_node_bytes(session, node_index); + const uint8_t* params = layout_node_params(session, node); + uint16_t operation; + uint32_t input_index; + uint32_t output_index; + const lw_runtime_tensor* input; + const lw_runtime_tensor* output; + if (node == NULL) { + return 0; + } + operation = lwm_read_u16(node); + input_index = lwm_read_u32(node + 8u); + output_index = lwm_read_u32(node + 40u); + switch (operation) { + case LW_OP_CONV: + return layout_conv_capable(session, node, params); + case LW_OP_CONV_TRANSPOSE: + return layout_conv_transpose_capable(session, node, params); + case LW_OP_ADD: + case LW_OP_MUL: + case LW_OP_DIV: + case LW_OP_SUB: + case LW_OP_POW: + case LW_OP_RELU: + case LW_OP_SIGMOID: + case LW_OP_SQRT: + case LW_OP_ERF: + case LW_OP_HARD_SIGMOID: + return layout_elementwise_capable(session, node, operation); + case LW_OP_REDUCE_MEAN: + return layout_reduce_mean_capable(session, node, params); + case LW_OP_CONCAT: + return layout_concat_capable(session, node, params); + case LW_OP_BATCH_NORMALIZATION: + if (lwm_read_u16(node + 2u) < 5u || lwm_read_u16(node + 4u) != 1u || + !layout_tensor_valid(session, input_index) || !layout_tensor_valid(session, output_index)) { + return 0; + } + input = &session->tensors[input_index]; + output = &session->tensors[output_index]; + return layout_rank4_f32(input) && layout_rank4_f32(output) && + layout_same_shape(input, output); + case LW_OP_AVERAGE_POOL: + case LW_OP_MAX_POOL: + if (params == NULL || lwm_read_u16(node + 2u) != 1u || lwm_read_u16(node + 4u) != 1u || + !layout_tensor_valid(session, input_index) || !layout_tensor_valid(session, output_index)) { + return 0; + } + input = &session->tensors[input_index]; + output = &session->tensors[output_index]; + return layout_rank4_f32(input) && layout_rank4_f32(output) && + lwm_read_i32(params + 8u) > 0 && lwm_read_i32(params + 12u) > 0 && + lwm_read_i32(params + 16u) > 0 && lwm_read_i32(params + 20u) > 0; + case LW_OP_RESIZE: + if (params == NULL || lwm_read_u16(node + 2u) != 1u || lwm_read_u16(node + 4u) != 1u || + !layout_tensor_valid(session, input_index) || !layout_tensor_valid(session, output_index) || + lwm_read_u16(params + 2u) != 4u) { + return 0; + } + input = &session->tensors[input_index]; + output = &session->tensors[output_index]; + { + float scale_n = layout_read_f32(params + 4u); + float scale_c = layout_read_f32(params + 8u); + float scale_h = layout_read_f32(params + 12u); + float scale_w = layout_read_f32(params + 16u); + return layout_rank4_f32(input) && layout_rank4_f32(output) && + isfinite(scale_n) && isfinite(scale_c) && isfinite(scale_h) && + isfinite(scale_w) && scale_n == 1.0f && scale_c == 1.0f && + layout_positive_integer_scale(scale_h) && + layout_positive_integer_scale(scale_w); + } + default: + return 0; + } +} + +static int layout_dense_conv_node(const lw_session* session, const uint8_t* node) { + const uint8_t* params = layout_node_params(session, node); + return node != NULL && lwm_read_u16(node) == LW_OP_CONV && params != NULL && + lwm_read_u32(params + 4u) == 1u; +} + +static int layout_node_activation_input(const lw_session* session, const uint8_t* node, + uint32_t input_slot, uint32_t* tensor_index) { + uint32_t input_count; + uint32_t candidate; + if (node == NULL || tensor_index == NULL) { + return 0; + } + input_count = lwm_read_u16(node + 2u); + if (input_slot >= input_count || input_slot >= LWM_V0_MAX_NODE_INPUTS) { + return 0; + } + candidate = lwm_read_u32(node + 8u + (size_t)input_slot * sizeof(uint32_t)); + if (!layout_tensor_valid(session, candidate) || layout_tensor_is_constant(session, candidate) || + lw_layout_tensor_is_neutral(session, candidate)) { + return 0; + } + *tensor_index = candidate; + return 1; +} +static int layout_node_starts_nhwc_segment(const lw_session* session, + uint32_t node_index) { + const uint8_t* node = layout_node_bytes(session, node_index); + const uint8_t* params = layout_node_params(session, node); + uint16_t operation; + if (node == NULL || !lw_layout_node_nhwc_capable(session, node_index)) { + return 0; + } + operation = lwm_read_u16(node); + if (operation == LW_OP_CONV_TRANSPOSE) { + return 1; + } + if (operation != LW_OP_CONV || params == NULL) { + return 0; + } + return lwm_read_u32(params + 4u) == 1u; +} + +static int layout_graph_input_can_start_nhwc(const lw_session* session, + uint32_t tensor_index) { + uint32_t node_index; + if (session == NULL || session->model == NULL) { + return 0; + } + for (node_index = 0u; node_index < session->model->info.node_count; ++node_index) { + const uint8_t* node = layout_node_bytes(session, node_index); + uint32_t input_count; + uint32_t input_slot; + if (node == NULL) { + continue; + } + input_count = lwm_read_u16(node + 2u); + if (input_count > LWM_V0_MAX_NODE_INPUTS) { + input_count = LWM_V0_MAX_NODE_INPUTS; + } + for (input_slot = 0u; input_slot < input_count; ++input_slot) { + uint32_t candidate = lwm_read_u32( + node + 8u + (size_t)input_slot * sizeof(uint32_t)); + if (candidate == tensor_index) { + return layout_node_starts_nhwc_segment(session, node_index); + } + } + } + return 0; +} +static void layout_plan_require_layout(lw_layout_plan* plan, + uint32_t tensor_index, + uint8_t required_layout) { + uint8_t required_flag; + if (plan == NULL || tensor_index >= plan->tensor_count) { + return; + } + required_flag = required_layout == LW_FAST_LAYOUT_NHWC + ? LW_LAYOUT_AVAILABLE_NHWC : LW_LAYOUT_AVAILABLE_NCHW; + if ((plan->tensor_available_layouts[tensor_index] & required_flag) != 0u) { + return; + } + plan->tensor_available_layouts[tensor_index] |= required_flag; + if (plan->layout_conversion_count != UINT32_MAX) { + ++plan->layout_conversion_count; + } +} +static void layout_plan_publish_output(const lw_session* session, lw_layout_plan* plan, + uint32_t output_index, uint8_t output_layout) { + if (session == NULL || plan == NULL || output_index >= plan->tensor_count) { + return; + } + if (lw_layout_tensor_is_neutral(session, output_index)) { + plan->tensor_layout[output_index] = LW_FAST_LAYOUT_NCHW; + plan->tensor_available_layouts[output_index] = + (uint8_t)(LW_LAYOUT_AVAILABLE_NCHW | LW_LAYOUT_AVAILABLE_NHWC); + } else if (output_layout == LW_FAST_LAYOUT_NHWC) { + plan->tensor_layout[output_index] = LW_FAST_LAYOUT_NHWC; + plan->tensor_available_layouts[output_index] = LW_LAYOUT_AVAILABLE_NHWC; + } else { + plan->tensor_layout[output_index] = LW_FAST_LAYOUT_NCHW; + plan->tensor_available_layouts[output_index] = LW_LAYOUT_AVAILABLE_NCHW; + } +} +void lw_layout_plan_free(lw_layout_plan* plan) { + if (plan == NULL) { + return; + } + free(plan->tensor_layout); + free(plan->tensor_available_layouts); + free(plan->node_layout); + memset(plan, 0, sizeof(*plan)); +} + +lw_status lw_layout_plan_build(const lw_session* session, + const lw_layout_planner_options* options, + lw_layout_plan* plan, lw_error* error) { + lw_layout_planner_options defaults; + uint32_t tensor_index; + uint32_t node_index; + int in_nhwc_island = 0; + if (session == NULL || session->model == NULL || session->tensors == NULL || plan == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "layout planner requires a session and plan"); + return LW_STATUS_INVALID_ARGUMENT; + } + memset(&defaults, 0, sizeof(defaults)); + if (options == NULL) { + options = &defaults; + } + lw_layout_plan_free(plan); + plan->tensor_count = session->model->info.tensor_count; + plan->node_count = session->model->info.node_count; + if (plan->tensor_count != 0u) { + plan->tensor_layout = (uint8_t*)calloc(plan->tensor_count, sizeof(uint8_t)); + plan->tensor_available_layouts = + (uint8_t*)calloc(plan->tensor_count, sizeof(uint8_t)); + } + if (plan->node_count != 0u) { + plan->node_layout = (uint8_t*)calloc(plan->node_count, sizeof(uint8_t)); + } + if ((plan->tensor_count != 0u && + (plan->tensor_layout == NULL || plan->tensor_available_layouts == NULL)) || + (plan->node_count != 0u && plan->node_layout == NULL)) { + lw_layout_plan_free(plan); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate layout plan"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (tensor_index = 0u; tensor_index < plan->tensor_count; ++tensor_index) { + plan->tensor_layout[tensor_index] = LW_FAST_LAYOUT_NCHW; + plan->tensor_available_layouts[tensor_index] = LW_LAYOUT_AVAILABLE_NCHW; + if ((session->tensors[tensor_index].flags & LWM_V0_TENSOR_FLAG_INPUT) != 0u && + options->allow_direct_nhwc_graph_input != 0u && + layout_graph_input_can_start_nhwc(session, tensor_index) && + layout_rank4_f32(&session->tensors[tensor_index]) && + !lw_layout_tensor_is_neutral(session, tensor_index)) { + plan->tensor_layout[tensor_index] = LW_FAST_LAYOUT_NHWC; + plan->tensor_available_layouts[tensor_index] = LW_LAYOUT_AVAILABLE_NHWC; + plan->graph_input_direct_nhwc = 1u; + } + } + for (node_index = 0u; node_index < plan->node_count; ++node_index) { + const uint8_t* node = layout_node_bytes(session, node_index); + const uint16_t operation = node == NULL ? 0u : lwm_read_u16(node); + const uint32_t output_index = node == NULL ? UINT32_MAX : lwm_read_u32(node + 40u); + const int capable = lw_layout_node_nhwc_capable(session, node_index); + int has_nhwc_input = 0; + int has_nchw_input = 0; + uint32_t input_slot; + int choose_nhwc = 0; + for (input_slot = 0u; node != NULL && input_slot < lwm_read_u16(node + 2u); ++input_slot) { + uint32_t input_index; + if (!layout_node_activation_input(session, node, input_slot, &input_index)) { + continue; + } + if ((plan->tensor_available_layouts[input_index] & LW_LAYOUT_AVAILABLE_NHWC) != 0u) { + has_nhwc_input = 1; + } + } + if (capable && (has_nhwc_input || layout_dense_conv_node(session, node) || + (operation == LW_OP_CONV_TRANSPOSE && has_nhwc_input))) { + choose_nhwc = 1; + } + if (choose_nhwc) { + plan->node_layout[node_index] = LW_FAST_LAYOUT_NHWC; + if (plan->nhwc_node_count != UINT32_MAX) { + ++plan->nhwc_node_count; + } + if (!in_nhwc_island && plan->nhwc_island_count != UINT32_MAX) { + ++plan->nhwc_island_count; + } + in_nhwc_island = 1; + for (input_slot = 0u; input_slot < lwm_read_u16(node + 2u); ++input_slot) { + uint32_t input_index; + if (layout_node_activation_input(session, node, input_slot, &input_index)) { + layout_plan_require_layout(plan, input_index, LW_FAST_LAYOUT_NHWC); + } + } + layout_plan_publish_output(session, plan, output_index, LW_FAST_LAYOUT_NHWC); + } else { + plan->node_layout[node_index] = LW_FAST_LAYOUT_NCHW; + if (plan->nchw_node_count != UINT32_MAX) { + ++plan->nchw_node_count; + } + in_nhwc_island = 0; + for (input_slot = 0u; node != NULL && input_slot < lwm_read_u16(node + 2u); ++input_slot) { + uint32_t input_index; + if (layout_node_activation_input(session, node, input_slot, &input_index)) { + layout_plan_require_layout(plan, input_index, LW_FAST_LAYOUT_NCHW); + } + } + layout_plan_publish_output(session, plan, output_index, LW_FAST_LAYOUT_NCHW); + } + (void)has_nchw_input; + } + for (tensor_index = 0u; tensor_index < plan->tensor_count; ++tensor_index) { + if ((session->tensors[tensor_index].flags & LWM_V0_TENSOR_FLAG_OUTPUT) != 0u && + !lw_layout_tensor_is_neutral(session, tensor_index) && + plan->tensor_layout[tensor_index] == LW_FAST_LAYOUT_NHWC) { + layout_plan_require_layout(plan, tensor_index, LW_FAST_LAYOUT_NCHW); + plan->tensor_layout[tensor_index] = LW_FAST_LAYOUT_NCHW; + } + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/layout_planner_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/layout_planner_internal.h new file mode 100644 index 00000000..4042b139 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/layout_planner_internal.h @@ -0,0 +1,42 @@ +#ifndef LW_LAYOUT_PLANNER_INTERNAL_H +#define LW_LAYOUT_PLANNER_INTERNAL_H + +#include "lw_infer.h" + +#include + +typedef enum lw_fast_layout { + LW_FAST_LAYOUT_NCHW = 0, + LW_FAST_LAYOUT_NHWC = 1 +} lw_fast_layout; + +enum { + LW_LAYOUT_AVAILABLE_NCHW = 1u << 0, + LW_LAYOUT_AVAILABLE_NHWC = 1u << 1 +}; + +typedef struct lw_layout_planner_options { + uint8_t allow_direct_nhwc_graph_input; +} lw_layout_planner_options; + +typedef struct lw_layout_plan { + uint32_t tensor_count; + uint32_t node_count; + uint8_t* tensor_layout; + uint8_t* tensor_available_layouts; + uint8_t* node_layout; + uint32_t nhwc_node_count; + uint32_t nchw_node_count; + uint32_t layout_conversion_count; + uint32_t nhwc_island_count; + uint8_t graph_input_direct_nhwc; +} lw_layout_plan; + +lw_status lw_layout_plan_build(const lw_session* session, + const lw_layout_planner_options* options, + lw_layout_plan* plan, lw_error* error); +void lw_layout_plan_free(lw_layout_plan* plan); +int lw_layout_tensor_is_neutral(const lw_session* session, uint32_t tensor_index); +int lw_layout_node_nhwc_capable(const lw_session* session, uint32_t node_index); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/lwm_read.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/lwm_read.h new file mode 100644 index 00000000..261d560e --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/lwm_read.h @@ -0,0 +1,24 @@ +#ifndef LW_LWM_READ_H +#define LW_LWM_READ_H + +/* Unaligned little-endian readers used instead of casting on-disk bytes. */ + +#include + +static inline uint16_t lwm_read_u16(const uint8_t* p) { + return (uint16_t)((uint16_t)p[0] | ((uint16_t)p[1] << 8)); +} + +static inline uint32_t lwm_read_u32(const uint8_t* p) { + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} + +static inline int32_t lwm_read_i32(const uint8_t* p) { + return (int32_t)lwm_read_u32(p); +} + +static inline uint64_t lwm_read_u64(const uint8_t* p) { + return (uint64_t)lwm_read_u32(p) | ((uint64_t)lwm_read_u32(p + 4) << 32); +} + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/memory.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/memory.c new file mode 100644 index 00000000..b92c748c --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/memory.c @@ -0,0 +1,432 @@ +#include "session_internal.h" + +/* + * Lifetime-aware workspace planner. + * Intermediate tensors whose lifetimes do not overlap may reuse the same byte + * range. This keeps inference allocation-free after session creation while the + * configured workspace ceiling remains a hard limit. + */ +#include "lwm_read.h" + +#include + +typedef struct free_block { + uint64_t offset; + uint64_t size; +} free_block; + +static uint64_t align_workspace(uint64_t value) { + return (value + (LW_WORKSPACE_ALIGNMENT - 1u)) & ~(uint64_t)(LW_WORKSPACE_ALIGNMENT - 1u); +} + +static int add_free_block(free_block* blocks, uint32_t* count, uint32_t capacity, uint64_t offset, + uint64_t size) { + uint32_t position = 0u; + uint32_t i; + if (*count >= capacity) { + return 0; + } + while (position < *count && blocks[position].offset < offset) { + ++position; + } + for (i = *count; i > position; --i) { + blocks[i] = blocks[i - 1u]; + } + blocks[position].offset = offset; + blocks[position].size = size; + ++(*count); + if (position > 0u && + blocks[position - 1u].offset + blocks[position - 1u].size == blocks[position].offset) { + blocks[position - 1u].size += blocks[position].size; + for (i = position; i + 1u < *count; ++i) { + blocks[i] = blocks[i + 1u]; + } + --(*count); + --position; + } + if (position + 1u < *count && + blocks[position].offset + blocks[position].size == blocks[position + 1u].offset) { + blocks[position].size += blocks[position + 1u].size; + for (i = position + 1u; i + 1u < *count; ++i) { + blocks[i] = blocks[i + 1u]; + } + --(*count); + } + return 1; +} + +static int allocate_block(free_block* blocks, uint32_t* count, uint64_t size, uint64_t* offset) { + uint32_t i; + uint32_t j; + for (i = 0u; i < *count; ++i) { + if (blocks[i].size >= size) { + *offset = blocks[i].offset; + blocks[i].offset += size; + blocks[i].size -= size; + if (blocks[i].size == 0u) { + for (j = i; j + 1u < *count; ++j) { + blocks[j] = blocks[j + 1u]; + } + --(*count); + } + return 1; + } + } + return 0; +} + +static lw_status lw_plan_workspace_v1(lw_session* session, uint64_t max_workspace_size, + lw_error* error) { + const lw_model* model = session->model; + free_block* blocks; + uint32_t free_count = 0u; + uint64_t workspace_end = 0u; + uint32_t i; + uint32_t tensor_index; + + blocks = (free_block*)calloc((size_t)model->info.tensor_count + 1u, sizeof(*blocks)); + if (blocks == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate workspace planner state"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (tensor_index = 0u; tensor_index < model->info.tensor_count; ++tensor_index) { + lw_runtime_tensor* tensor = &session->tensors[tensor_index]; + tensor->workspace_offset = UINT64_MAX; + tensor->birth_node = -1; + tensor->last_use_node = -1; + tensor->workspace_live = 0; + } + /* First determine when each tensor is born and when its final consumer + * runs. Graph outputs stay live until execution has completely finished. */ + for (i = 0u; i < model->info.node_count; ++i) { + const uint8_t* node = + model->bytes + (size_t)model->node_offset + (size_t)i * LWM_V0_NODE_SIZE; + uint16_t input_count = lwm_read_u16(node + 2); + uint16_t output_count = lwm_read_u16(node + 4); + uint32_t j; + for (j = 0u; j < input_count; ++j) { + lw_runtime_tensor* tensor = &session->tensors[lwm_read_u32(node + 8u + j * 4u)]; + tensor->last_use_node = (int32_t)i; + } + for (j = 0u; j < output_count; ++j) { + lw_runtime_tensor* tensor = &session->tensors[lwm_read_u32(node + 40u + j * 4u)]; + if (tensor->birth_node != -1) { + free(blocks); + lw_set_error(error, LW_STATUS_INVALID_FORMAT, + "multiple nodes produce the same tensor"); + return LW_STATUS_INVALID_FORMAT; + } + tensor->birth_node = (int32_t)i; + tensor->last_use_node = (int32_t)i; + } + } + for (i = 0u; i < model->info.output_count; ++i) { + uint32_t index = lwm_read_u32(model->bytes + (size_t)model->output_offset + (size_t)i * 4u); + session->tensors[index].last_use_node = (int32_t)model->info.node_count; + } + + /* Walking nodes in execution order lets an expired input range be reused + * immediately by a later output instead of growing the workspace. */ + for (i = 0u; i < model->info.node_count; ++i) { + const uint8_t* node = + model->bytes + (size_t)model->node_offset + (size_t)i * LWM_V0_NODE_SIZE; + uint16_t output_count = lwm_read_u16(node + 4); + uint32_t j; + for (tensor_index = 0u; tensor_index < model->info.tensor_count; ++tensor_index) { + lw_runtime_tensor* tensor = &session->tensors[tensor_index]; + if (tensor->workspace_live && tensor->last_use_node < (int32_t)i) { + uint64_t size = align_workspace(tensor->byte_size); + if (!add_free_block(blocks, &free_count, model->info.tensor_count + 1u, + tensor->workspace_offset, size)) { + free(blocks); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, + "workspace free-list capacity exceeded"); + return LW_STATUS_OUT_OF_MEMORY; + } + tensor->workspace_live = 0; + } + } + for (j = 0u; j < output_count; ++j) { + uint32_t output_index = lwm_read_u32(node + 40u + j * 4u); + lw_runtime_tensor* tensor = &session->tensors[output_index]; + uint64_t size; + if (lw_execution_plan_is_skipped_tensor(session, output_index)) { + continue; + } + uint64_t offset; + if (tensor->byte_size > UINT64_MAX - (LW_WORKSPACE_ALIGNMENT - 1u)) { + free(blocks); + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, + "aligned tensor workspace size overflows"); + return LW_STATUS_MEMORY_LIMIT; + } + size = align_workspace(tensor->byte_size); + if (!allocate_block(blocks, &free_count, size, &offset)) { + if (workspace_end > UINT64_MAX - size) { + free(blocks); + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, "workspace size overflows"); + return LW_STATUS_MEMORY_LIMIT; + } + offset = workspace_end; + workspace_end += size; + } + if (workspace_end > max_workspace_size || workspace_end > SIZE_MAX) { + free(blocks); + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, + "planned workspace exceeds max_workspace_size"); + return LW_STATUS_MEMORY_LIMIT; + } + tensor->workspace_offset = offset; + tensor->workspace_live = 1; + } + } + free(blocks); + session->workspace_bytes = (size_t)workspace_end; + return LW_STATUS_OK; +} + +#if defined(LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2) +typedef struct workspace_interval { + uint32_t tensor_index; + uint64_t size; + uint64_t offset; + int32_t birth_node; + int32_t last_use_node; + int placed; +} workspace_interval; + +static int interval_lifetimes_overlap(const workspace_interval* left, + const workspace_interval* right) { + return left->birth_node <= right->last_use_node && + right->birth_node <= left->last_use_node; +} + +static int interval_before(const workspace_interval* left, const workspace_interval* right) { + if (left->size != right->size) { + return left->size > right->size; + } + if (left->birth_node != right->birth_node) { + return left->birth_node < right->birth_node; + } + return left->tensor_index < right->tensor_index; +} + +static int align_workspace_checked(uint64_t value, uint64_t* aligned) { + if (value > UINT64_MAX - (LW_WORKSPACE_ALIGNMENT - 1u)) { + return 0; + } + *aligned = align_workspace(value); + return 1; +} + +static int interval_find_offset(const workspace_interval* intervals, uint32_t count, + uint32_t current, uint64_t* out_offset) { + uint64_t candidate = 0u; + for (;;) { + uint32_t i; + int moved = 0; + uint64_t candidate_end; + if (intervals[current].size > UINT64_MAX - candidate) { + return 0; + } + candidate_end = candidate + intervals[current].size; + for (i = 0u; i < count; ++i) { + const workspace_interval* other = &intervals[i]; + uint64_t other_end; + if (!other->placed || !interval_lifetimes_overlap(&intervals[current], other)) { + continue; + } + if (other->size > UINT64_MAX - other->offset) { + return 0; + } + other_end = other->offset + other->size; + if (candidate < other_end && other->offset < candidate_end) { + if (!align_workspace_checked(other_end, &candidate)) { + return 0; + } + moved = 1; + break; + } + } + if (!moved) { + *out_offset = candidate; + return 1; + } + } +} + +static lw_status lw_plan_workspace_v2(lw_session* session, uint64_t max_workspace_size, + lw_error* error) { + const lw_model* model = session->model; + workspace_interval* intervals; + uint32_t interval_count = 0u; + uint32_t i; + uint64_t workspace_end = 0u; + + intervals = (workspace_interval*)calloc((size_t)model->info.tensor_count, + sizeof(*intervals)); + if (intervals == NULL && model->info.tensor_count != 0u) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, + "unable to allocate v2 workspace planner state"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (i = 0u; i < model->info.tensor_count; ++i) { + lw_runtime_tensor* tensor = &session->tensors[i]; + tensor->workspace_offset = UINT64_MAX; + tensor->birth_node = -1; + tensor->last_use_node = -1; + tensor->workspace_live = 0; + } + /* Recompute the semantic lifetimes independently of the v1 planner. */ + for (i = 0u; i < model->info.node_count; ++i) { + const uint8_t* node = + model->bytes + (size_t)model->node_offset + (size_t)i * LWM_V0_NODE_SIZE; + uint16_t input_count = lwm_read_u16(node + 2u); + uint16_t output_count = lwm_read_u16(node + 4u); + uint32_t j; + for (j = 0u; j < input_count; ++j) { + uint32_t index = lwm_read_u32(node + 8u + (size_t)j * sizeof(uint32_t)); + session->tensors[index].last_use_node = (int32_t)i; + } + for (j = 0u; j < output_count; ++j) { + uint32_t index = lwm_read_u32(node + 40u + (size_t)j * sizeof(uint32_t)); + lw_runtime_tensor* tensor = &session->tensors[index]; + if (tensor->birth_node != -1) { + free(intervals); + lw_set_error(error, LW_STATUS_INVALID_FORMAT, + "multiple nodes produce the same tensor"); + return LW_STATUS_INVALID_FORMAT; + } + tensor->birth_node = (int32_t)i; + tensor->last_use_node = (int32_t)i; + if (lw_execution_plan_is_skipped_tensor(session, index)) { + continue; + } + intervals[interval_count].tensor_index = index; + intervals[interval_count].birth_node = (int32_t)i; + intervals[interval_count].last_use_node = (int32_t)i; + if (!align_workspace_checked(tensor->byte_size, &intervals[interval_count].size)) { + free(intervals); + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, + "aligned tensor workspace size overflows"); + return LW_STATUS_MEMORY_LIMIT; + } + ++interval_count; + } + } + for (i = 0u; i < model->info.output_count; ++i) { + uint32_t index = lwm_read_u32(model->bytes + (size_t)model->output_offset + + (size_t)i * sizeof(uint32_t)); + session->tensors[index].last_use_node = (int32_t)model->info.node_count; + } + for (i = 0u; i < interval_count; ++i) { + intervals[i].last_use_node = session->tensors[intervals[i].tensor_index].last_use_node; + } + /* Stable insertion sort keeps the plan deterministic across compilers. */ + for (i = 1u; i < interval_count; ++i) { + workspace_interval value = intervals[i]; + uint32_t position = i; + while (position > 0u && interval_before(&value, &intervals[position - 1u])) { + intervals[position] = intervals[position - 1u]; + --position; + } + intervals[position] = value; + } + for (i = 0u; i < interval_count; ++i) { + workspace_interval* interval = &intervals[i]; + uint64_t end; + if (interval->size == 0u) { + interval->offset = 0u; + } else if (!interval_find_offset(intervals, interval_count, i, &interval->offset) || + interval->size > UINT64_MAX - interval->offset) { + free(intervals); + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, + "v2 workspace interval placement overflows"); + return LW_STATUS_MEMORY_LIMIT; + } + interval->placed = 1; + end = interval->offset + interval->size; + if (end > workspace_end) { + workspace_end = end; + } + } + if (workspace_end > max_workspace_size || workspace_end > SIZE_MAX) { + free(intervals); + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, + "planned v2 workspace exceeds max_workspace_size"); + return LW_STATUS_MEMORY_LIMIT; + } + for (i = 0u; i < interval_count; ++i) { + lw_runtime_tensor* tensor = &session->tensors[intervals[i].tensor_index]; + tensor->workspace_offset = intervals[i].offset; + tensor->workspace_live = intervals[i].size != 0u; + } + free(intervals); + session->workspace_bytes = (size_t)workspace_end; + return LW_STATUS_OK; +} +#endif + +lw_status lw_plan_workspace(lw_session* session, uint64_t max_workspace_size, lw_error* error) { +#if defined(LW_EXPERIMENTAL_WORKSPACE_PLANNER_V2) + const uint32_t tensor_count = session->model->info.tensor_count; + uint64_t* v1_offsets = + (uint64_t*)calloc((size_t)tensor_count, sizeof(*v1_offsets)); + unsigned char* v1_live = + (unsigned char*)calloc((size_t)tensor_count, sizeof(*v1_live)); + uint64_t v1_workspace = 0u; + lw_error v1_error; + lw_error v2_error; + lw_status v1_status; + lw_status v2_status; + uint32_t i; + if ((v1_offsets == NULL || v1_live == NULL) && tensor_count != 0u) { + free(v1_offsets); + free(v1_live); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, + "unable to allocate workspace planner comparison state"); + return LW_STATUS_OUT_OF_MEMORY; + } + lw_error_init(&v1_error); + v1_status = lw_plan_workspace_v1(session, max_workspace_size, &v1_error); + if (v1_status == LW_STATUS_OK) { + v1_workspace = (uint64_t)session->workspace_bytes; + for (i = 0u; i < tensor_count; ++i) { + v1_offsets[i] = session->tensors[i].workspace_offset; + v1_live[i] = (unsigned char)(session->tensors[i].workspace_live != 0); + } + } + lw_error_init(&v2_error); + v2_status = lw_plan_workspace_v2(session, max_workspace_size, &v2_error); + if (v2_status == LW_STATUS_OK && + (v1_status != LW_STATUS_OK || (uint64_t)session->workspace_bytes <= v1_workspace)) { + free(v1_offsets); + free(v1_live); + if (error != NULL) { + *error = v2_error; + } + return LW_STATUS_OK; + } + if (v1_status == LW_STATUS_OK) { + session->workspace_bytes = (size_t)v1_workspace; + for (i = 0u; i < tensor_count; ++i) { + session->tensors[i].workspace_offset = v1_offsets[i]; + session->tensors[i].workspace_live = v1_live[i] != 0u; + } + free(v1_offsets); + free(v1_live); + if (error != NULL) { + *error = v1_error; + } + return LW_STATUS_OK; + } + free(v1_offsets); + free(v1_live); + if (error != NULL) { + *error = v2_status == LW_STATUS_OK ? v2_error : v1_error; + } + return v2_status == LW_STATUS_OK ? LW_STATUS_OK : v1_status; +#else + return lw_plan_workspace_v1(session, max_workspace_size, error); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/model.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/model.c new file mode 100644 index 00000000..8dbc806d --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/model.c @@ -0,0 +1,238 @@ +#include "model_internal.h" +#include "abi_compat_internal.h" + +/* + * Model handle lifecycle and platform-specific UTF-8 file opening. + * A model is published only after the untrusted LWM/ONNX stream is validated. + * ONNX import normalizes to the same checked IR consumed by validate.c. + */ + +#include +#include +#include + +#if defined(_WIN32) +# define WIN32_LEAN_AND_MEAN +# include +#endif + +#define LW_DEFAULT_MAX_FILE_SIZE (UINT64_C(1024) * UINT64_C(1024) * UINT64_C(1024)) + +void lw_set_error(lw_error* error, lw_status status, const char* message) { + if (error == NULL) { + return; + } + error->struct_size = (uint32_t)sizeof(*error); + error->code = (int32_t)status; + if (message == NULL) { + error->message[0] = '\0'; + return; + } +#if defined(_MSC_VER) + (void)strncpy_s(error->message, sizeof(error->message), message, _TRUNCATE); +#else + (void)snprintf(error->message, sizeof(error->message), "%s", message); +#endif +} + +void lw_model_options_init(lw_model_options* options) { + if (options == NULL) { + return; + } + memset(options, 0, sizeof(*options)); + options->struct_size = (uint32_t)sizeof(*options); + options->max_file_size = LW_DEFAULT_MAX_FILE_SIZE; +} + +void lw_error_init(lw_error* error) { + if (error == NULL) { + return; + } + memset(error, 0, sizeof(*error)); + error->struct_size = (uint32_t)sizeof(*error); +} + +void lw_model_info_init(lw_model_info* info) { + if (info == NULL) { + return; + } + memset(info, 0, sizeof(*info)); + info->struct_size = (uint32_t)sizeof(*info); +} + +#if defined(_WIN32) +static FILE* lw_open_read_utf8(const char* path_utf8) { + int wide_count; + wchar_t* wide_path; + FILE* file = NULL; + wide_count = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path_utf8, -1, NULL, 0); + if (wide_count <= 0) { + return NULL; + } + wide_path = (wchar_t*)malloc((size_t)wide_count * sizeof(wchar_t)); + if (wide_path == NULL) { + return NULL; + } + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path_utf8, -1, wide_path, wide_count) > + 0) { + if (_wfopen_s(&file, wide_path, L"rb") != 0) { + file = NULL; + } + } + free(wide_path); + return file; +} +#else +static FILE* lw_open_read_utf8(const char* path_utf8) { + return fopen(path_utf8, "rb"); +} +#endif + +lw_status lw_model_load(const char* path_utf8, const lw_model_options* options, + lw_model** out_model, lw_error* error) { + FILE* file; + long length; + uint64_t limit = LW_DEFAULT_MAX_FILE_SIZE; + lw_model* model; + size_t read_count; + lw_status status; + uint8_t magic[4]; + + if (out_model != NULL) { + *out_model = NULL; + } + if (path_utf8 == NULL || path_utf8[0] == '\0' || out_model == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "path and out_model are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (options != NULL) { + if (options->struct_size != sizeof(*options) || options->reserved != 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid model options structure"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (options->max_file_size != 0u) { + limit = options->max_file_size; + } + } + file = lw_open_read_utf8(path_utf8); + if (file == NULL) { + lw_set_error(error, LW_STATUS_IO_ERROR, "unable to open model file"); + return LW_STATUS_IO_ERROR; + } + if (fseek(file, 0, SEEK_END) != 0 || (length = ftell(file)) < 0 || + fseek(file, 0, SEEK_SET) != 0) { + fclose(file); + lw_set_error(error, LW_STATUS_IO_ERROR, "unable to determine model file size"); + return LW_STATUS_IO_ERROR; + } + if ((uint64_t)length > limit || (uint64_t)length > (uint64_t)SIZE_MAX) { + fclose(file); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "model file exceeds configured size limit"); + return LW_STATUS_OUT_OF_BOUNDS; + } + if ((uint64_t)length < LWM_V0_HEADER_SIZE) { + fclose(file); + lw_set_error(error, LW_STATUS_INVALID_FORMAT, "model file is smaller than the minimum supported size"); + return LW_STATUS_INVALID_FORMAT; + } + if (fread(magic, 1u, sizeof(magic), file) != sizeof(magic) || + fseek(file, 0, SEEK_SET) != 0) { + fclose(file); + lw_set_error(error, LW_STATUS_IO_ERROR, "unable to read model signature"); + return LW_STATUS_IO_ERROR; + } + if (memcmp(magic, "LWM0", sizeof(magic)) != 0 && (uint64_t)length > LW_ONNX_MAX_BYTES) { + fclose(file); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "ONNX file exceeds importer size limit"); + return LW_STATUS_OUT_OF_BOUNDS; + } + model = (lw_model*)calloc(1u, sizeof(*model)); + if (model == NULL) { + fclose(file); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate model handle"); + return LW_STATUS_OUT_OF_MEMORY; + } + model->ref_count = 1u; + model->byte_count = (size_t)length; + model->bytes = (uint8_t*)malloc(model->byte_count); + if (model->bytes == NULL) { + fclose(file); + free(model); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate model buffer"); + return LW_STATUS_OUT_OF_MEMORY; + } + read_count = fread(model->bytes, 1u, model->byte_count, file); + fclose(file); + if (read_count != model->byte_count) { + lw_model_free(model); + lw_set_error(error, LW_STATUS_IO_ERROR, "unable to read complete model file"); + return LW_STATUS_IO_ERROR; + } + /* Do not expose even a successfully read file until its complete binary + * structure, operator subset, checksums and table ranges are trusted. */ + status = memcmp(model->bytes, "LWM0", 4u) == 0 + ? lw_validate_lwm_v0(model, error) : lw_import_onnx(model, error); + if (status != LW_STATUS_OK) { + lw_model_free(model); + return status; + } + *out_model = model; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +void lw_model_free(lw_model* model) { + if (model == NULL) { + return; + } + if (model->ref_count > 1u) { + --model->ref_count; + return; + } + free(model->bytes); + model->bytes = NULL; + free(model); +} + +void lw_model_retain(lw_model* model) { + if (model != NULL && model->ref_count != UINT32_MAX) { + ++model->ref_count; + } +} + +lw_status lw_model_get_info(const lw_model* model, lw_model_info* info) { + if (model == NULL || info == NULL || + !lw_abi_copy_output_prefix(info, info->struct_size, &model->info, sizeof(model->info))) { + return LW_STATUS_INVALID_ARGUMENT; + } + return LW_STATUS_OK; +} + +const char* lw_status_string(lw_status status) { + switch (status) { + case LW_STATUS_OK: + return "ok"; + case LW_STATUS_INVALID_ARGUMENT: + return "invalid_argument"; + case LW_STATUS_IO_ERROR: + return "io_error"; + case LW_STATUS_OUT_OF_MEMORY: + return "out_of_memory"; + case LW_STATUS_INVALID_FORMAT: + return "invalid_format"; + case LW_STATUS_UNSUPPORTED_VERSION: + return "unsupported_version"; + case LW_STATUS_OUT_OF_BOUNDS: + return "out_of_bounds"; + case LW_STATUS_CHECKSUM_MISMATCH: + return "checksum_mismatch"; + case LW_STATUS_UNSUPPORTED: + return "unsupported"; + case LW_STATUS_INVALID_SHAPE: + return "invalid_shape"; + case LW_STATUS_MEMORY_LIMIT: + return "memory_limit"; + default: + return "unknown"; + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/model_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/model_internal.h new file mode 100644 index 00000000..5315e6bd --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/model_internal.h @@ -0,0 +1,45 @@ +#ifndef LW_MODEL_INTERNAL_H +#define LW_MODEL_INTERNAL_H + +/* Private in-memory representation of a validated LWM model. */ + +#include "error_internal.h" +#include "lw_infer.h" + +#include +#include + +#define LWM_V0_HEADER_SIZE 160u +#define LWM_V0_TENSOR_SIZE 80u +#define LWM_V0_NODE_SIZE 72u +#define LWM_V0_CHECKSUM_OFFSET 128u +#define LWM_V0_MAX_DIMS 8u +#define LWM_V0_MAX_NODE_INPUTS 8u +#define LWM_V0_MAX_NODE_OUTPUTS 4u +#define LWM_V0_NO_WORKSPACE UINT64_MAX +#define LWM_V0_HEADER_FLAG_NO_MEMORY_PLAN 1u +#define LWM_V0_TENSOR_FLAG_CONSTANT 1u +#define LWM_V0_TENSOR_FLAG_INPUT 2u +#define LWM_V0_TENSOR_FLAG_OUTPUT 4u + +struct lw_model { + uint32_t ref_count; + uint8_t* bytes; + size_t byte_count; + lw_model_info info; + uint64_t input_offset; + uint64_t output_offset; + uint64_t tensor_offset; + uint64_t node_offset; + uint64_t param_offset; + uint64_t param_size; + uint64_t weight_offset; + uint64_t weight_size; +}; + +lw_status lw_validate_lwm_v0(lw_model* model, lw_error* error); +#define LW_ONNX_MAX_BYTES (UINT64_C(256) * 1024u * 1024u) +lw_status lw_import_onnx(lw_model* model, lw_error* error); +void lw_model_retain(lw_model* model); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/onnx_import.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/onnx_import.c new file mode 100644 index 00000000..0764b9f4 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/onnx_import.c @@ -0,0 +1,751 @@ +/* Pure-C bounded protobuf reader, inspired by lw.PPOCR.Vulkan (Apache-2.0). + * Targets our existing NCHW semantic IR, preserving LWM compatibility. */ +#include "session_internal.h" +#include "lwm_read.h" +#include +#include +#include +#include +#include +#include +#define OX_MAX_VALUES 16384u +#define OX_MAX_NODES 4096u +#define OX_MAX_ATTRS 16u +#define OX_MAX_BYTES LW_ONNX_MAX_BYTES +#define OX_NONE UINT32_MAX +typedef struct ox_reader { const uint8_t* p; const uint8_t* end; } ox_reader; +typedef struct ox_attr { + char name[64], string[128]; + int64_t ints[8], i; + uint32_t count, type; + float f; +} ox_attr; +typedef struct ox_value { + char* name; + uint32_t alias, defined, constant, ready, metadata, used, index, first_rank; + lw_runtime_tensor shape; + int32_t first[8], dynamic[8]; + const uint8_t* raw; + uint8_t* owned; + size_t bytes; + int64_t controls[8]; + uint32_t control_count; +} ox_value; +typedef struct ox_node { + char op[48]; + uint32_t inputs[8], output, count, attr_count, emitted; + ox_attr attrs[OX_MAX_ATTRS]; + uint8_t params[136]; + uint32_t param_size, lowered_count; + uint16_t opcode; +} ox_node; +typedef struct ox_graph { + ox_value* values; + ox_node* nodes; + uint32_t value_count, node_count, value_capacity, node_capacity, input, output, opset; + size_t names_bytes; + uint64_t constant_bytes; + lw_runtime_tensor declared_output; + jmp_buf* escape; + lw_error* error; + lw_status status; +} ox_graph; +/* All parse failures unwind to one cleanup point. Buffers tracked in ox_graph + * remain owned there until normalization succeeds; no partial model escapes. */ +#if defined(_MSC_VER) +__declspec(noreturn) +#else +_Noreturn +#endif +static void ox_bad(ox_graph* g,lw_status status,const char* message) { + g->status=status; lw_set_error(g->error,status,message); longjmp(*g->escape,1); +} +#define OX_REQUIRE(g,c,m) do { if(!(c)) ox_bad((g),LW_STATUS_INVALID_FORMAT,(m)); } while(0) +static uint64_t ox_var(ox_graph* g,ox_reader* r) { + uint64_t v=0u; unsigned i; + for(i=0u;i<10u;++i) { + uint8_t b; + OX_REQUIRE(g,r->p!=r->end,"ONNX truncated varint"); b=*r->p++; + OX_REQUIRE(g,i!=9u || b<=1u,"ONNX varint overflow"); + v|=(uint64_t)(b&127u)<<(7u*i); + if((b&128u)==0u) return v; + } + ox_bad(g,LW_STATUS_INVALID_FORMAT,"ONNX invalid varint"); +} +static ox_reader ox_child(ox_graph* g,ox_reader* r) { + uint64_t n=ox_var(g,r); ox_reader c; + OX_REQUIRE(g,n<=(uint64_t)(r->end-r->p),"ONNX length exceeds message"); + c.p=r->p; c.end=r->p+(size_t)n; r->p=c.end; return c; +} +static uint32_t ox_fixed(ox_graph* g,ox_reader* r) { + uint32_t v; OX_REQUIRE(g,r->end-r->p>=4,"ONNX truncated fixed32"); + v=lwm_read_u32(r->p); r->p+=4; return v; +} +static int ox_tag(ox_graph* g,ox_reader* r,unsigned* f,unsigned* w) { + uint64_t t; if(r->p==r->end) return 0; t=ox_var(g,r); + OX_REQUIRE(g,t<=UINT32_MAX && (t>>3u)!=0u,"ONNX invalid tag"); + *f=(unsigned)(t>>3u); *w=(unsigned)(t&7u); return 1; +} +static void ox_skip(ox_graph* g,ox_reader* r,unsigned w) { + if(w==0u) { (void)ox_var(g,r); return; } + if(w==2u) { (void)ox_child(g,r); return; } + if(w==1u || w==5u) { + size_t n=w==1u?8u:4u; + OX_REQUIRE(g,(size_t)(r->end-r->p)>=n,"ONNX truncated fixed field"); r->p+=n; return; + } + ox_bad(g,LW_STATUS_INVALID_FORMAT,"ONNX unsupported wire type"); +} +static void ox_string(ox_graph* g,ox_reader* r,char* dst,size_t cap) { + ox_reader c=ox_child(g,r); size_t n=(size_t)(c.end-c.p); + OX_REQUIRE(g,n1u,"ONNX empty tensor name"); + for(i=0u;ivalue_count;++i) if(strcmp(g->values[i].name,name)==0) return i; + OX_REQUIRE(g,g->value_countvalue_capacity && n<=8u*1024u*1024u-g->names_bytes,"ONNX name budget"); + i=g->value_count++; g->values[i].name=(char*)malloc(n); + if(g->values[i].name==NULL) ox_bad(g,LW_STATUS_OUT_OF_MEMORY,"ONNX name allocation"); + memcpy(g->values[i].name,name,n); g->names_bytes+=n; g->values[i].alias=OX_NONE; return i; +} +static uint32_t ox_root(ox_graph* g,uint32_t id) { + uint32_t steps=0u; + while(g->values[id].alias!=OX_NONE) { + OX_REQUIRE(g,++steps<=g->value_count,"ONNX Identity cycle"); id=g->values[id].alias; + } + return id; +} +static void ox_put16(uint8_t* p,uint16_t v) { p[0]=(uint8_t)v; p[1]=(uint8_t)(v>>8); } +static void ox_put32(uint8_t* p,uint32_t v) { unsigned i; for(i=0u;i<4u;++i) p[i]=(uint8_t)(v>>(8u*i)); } +static void ox_put64(uint8_t* p,uint64_t v) { unsigned i; for(i=0u;i<8u;++i) p[i]=(uint8_t)(v>>(8u*i)); } +static void ox_float(uint8_t* p,float v) { uint32_t b; memcpy(&b,&v,4u); ox_put32(p,b); } +static uint32_t ox_dtype(ox_graph* g,uint32_t t) { + if(t==1u) return LW_DTYPE_F32; + if(t==6u) return LW_DTYPE_I32; + if(t==7u) return LW_DTYPE_I64; + if(t==2u) return LW_DTYPE_U8; + ox_bad(g,LW_STATUS_UNSUPPORTED,"ONNX unsupported dtype"); +} +/* Initializers borrow raw bytes during import. Only typed payloads need a + * temporary allocation; the normalized IR owns its final aligned copy. */ +static void ox_tensor(ox_graph* g,ox_reader r) { + char name[4097]={0}; int32_t dims[8]={0}; + uint32_t rank=0u,type=0u,id,i; const uint8_t* raw=NULL; + ox_reader typed={NULL,NULL}; unsigned typed_field=0u,f,w; + size_t raw_size=0u; uint64_t count=1u,bytes; ox_value* v; + while(ox_tag(g,&r,&f,&w)) { + if(f==1u && (w==0u || w==2u)) { + ox_reader c=w==2u?ox_child(g,&r):r; + while(c.p!=c.end) { + uint64_t d=ox_var(g,&c); + OX_REQUIRE(g,rank<8u && d<=INT32_MAX,"ONNX initializer dimensions"); + dims[rank++]=(int32_t)d; + if(w==0u) break; + } + if(w==0u) r=c; + } else if(f==2u && w==0u) { + uint64_t d=ox_var(g,&r); + OX_REQUIRE(g,type==0u && d<=UINT32_MAX,"ONNX duplicate/invalid dtype"); type=(uint32_t)d; + } else if(f==8u && w==2u) ox_string(g,&r,name,sizeof(name)); + else if(f==9u && w==2u) { + ox_reader c=ox_child(g,&r); + OX_REQUIRE(g,raw==NULL,"ONNX duplicate raw_data"); raw=c.p; raw_size=(size_t)(c.end-c.p); + } else if((f==4u || f==5u || f==7u) && w==2u) { + OX_REQUIRE(g,typed.p==NULL,"ONNX repeated typed payload"); + typed=ox_child(g,&r); typed_field=f; + } else if(f==14u && w==0u) { + OX_REQUIRE(g,ox_var(g,&r)==0u,"ONNX external tensor data unsupported"); + } else if(f==13u || f==14u) + ox_bad(g,LW_STATUS_UNSUPPORTED,"ONNX external tensor data unsupported"); + else ox_skip(g,&r,w); + } + id=ox_value_id(g,name); v=&g->values[id]; + OX_REQUIRE(g,!v->defined,"ONNX duplicate initializer/producer"); + v->defined=1u; v->constant=1u; v->shape.dtype=ox_dtype(g,type); v->shape.rank=rank; + memcpy(v->shape.dimensions,dims,sizeof(dims)); + for(i=0u;iconstant_bytes,"ONNX constant budget"); + g->constant_bytes+=bytes; v->bytes=(size_t)bytes; + if(raw!=NULL) { + OX_REQUIRE(g,typed.p==NULL && raw_size==v->bytes,"ONNX raw tensor size mismatch"); v->raw=raw; + } else if(bytes!=0u) { + size_t j; + OX_REQUIRE(g,typed.p!=NULL,"ONNX missing tensor data"); + v->owned=(uint8_t*)malloc(v->bytes); + if(v->owned==NULL) ox_bad(g,LW_STATUS_OUT_OF_MEMORY,"ONNX tensor allocation"); + v->raw=v->owned; + for(j=0u;j<(size_t)count;++j) { + if(type==1u) { + OX_REQUIRE(g,typed_field==4u,"ONNX float payload mismatch"); + ox_put32(v->owned+j*4u,ox_fixed(g,&typed)); + } else { + uint64_t x; + OX_REQUIRE(g,typed_field==(type==7u?7u:5u),"ONNX integer payload mismatch"); + x=ox_var(g,&typed); + if(type==7u) ox_put64(v->owned+j*8u,x); + else if(type==2u) { + OX_REQUIRE(g,x<=UINT8_MAX,"ONNX uint8 payload overflow"); + v->owned[j]=(uint8_t)x; + } else { + OX_REQUIRE(g,(int64_t)x>=INT32_MIN && (int64_t)x<=INT32_MAX, + "ONNX int32 payload overflow"); + ox_put32(v->owned+j*4u,(uint32_t)x); + } + } + } + OX_REQUIRE(g,typed.p==typed.end,"ONNX surplus typed data"); + } + if(type==1u) { + size_t j; + for(j=0u;jbytes;j+=4u) { + uint32_t b=lwm_read_u32(v->raw+j); float x; memcpy(&x,&b,4u); + OX_REQUIRE(g,isfinite(x),"ONNX nonfinite constant"); + } + } + if((type==6u || type==7u) && count<=8u) { + v->metadata=1u; v->control_count=(uint32_t)count; + for(i=0u;i<(uint32_t)count;++i) + v->controls[i]=type==7u?(int64_t)lwm_read_u64(v->raw+i*8u):(int32_t)lwm_read_u32(v->raw+i*4u); + } +} +static void ox_attribute(ox_graph* g,ox_reader r,ox_attr* a) { + unsigned f,w; + while(ox_tag(g,&r,&f,&w)) { + if(f==1u && w==2u) ox_string(g,&r,a->name,sizeof(a->name)); + else if(f==20u && w==0u) { + uint64_t t=ox_var(g,&r); OX_REQUIRE(g,t<=UINT32_MAX,"ONNX attribute type overflow"); a->type=(uint32_t)t; + } else if(f==2u && w==5u) { + uint32_t b=ox_fixed(g,&r); memcpy(&a->f,&b,4u); + OX_REQUIRE(g,isfinite(a->f),"ONNX nonfinite attribute"); + } else if(f==3u && w==0u) a->i=(int64_t)ox_var(g,&r); + else if(f==4u && w==2u) ox_string(g,&r,a->string,sizeof(a->string)); + else if(f==8u && (w==0u || w==2u)) { + ox_reader c=w==2u?ox_child(g,&r):r; + while(c.p!=c.end) { + OX_REQUIRE(g,a->count<8u,"ONNX attribute list budget"); + a->ints[a->count++]=(int64_t)ox_var(g,&c); + if(w==0u) break; + } + if(w==0u) r=c; + } else if(f==5u || f==6u || f==10u || f==11u) + ox_bad(g,LW_STATUS_UNSUPPORTED,"ONNX tensor/graph attribute unsupported"); + else ox_skip(g,&r,w); + } + OX_REQUIRE(g,a->name[0]!=0 && (a->type==1u || a->type==2u || a->type==3u || a->type==7u), + "ONNX unsupported attribute type"); +} +static void ox_parse_node(ox_graph* g,ox_reader r) { + unsigned f,w; char name[4097]; ox_node* n; uint32_t i; + OX_REQUIRE(g,g->node_countnode_capacity,"ONNX node budget"); + n=&g->nodes[g->node_count++]; n->output=OX_NONE; + while(ox_tag(g,&r,&f,&w)) { + if(f==1u && w==2u) { + OX_REQUIRE(g,n->count<8u,"ONNX input arity budget"); + ox_string(g,&r,name,sizeof(name)); n->inputs[n->count++]=name[0]==0?OX_NONE:ox_value_id(g,name); + } else if(f==2u && w==2u) { + OX_REQUIRE(g,n->output==OX_NONE,"ONNX multiple node outputs unsupported"); + ox_string(g,&r,name,sizeof(name)); n->output=ox_value_id(g,name); + } else if(f==4u && w==2u) ox_string(g,&r,n->op,sizeof(n->op)); + else if(f==5u && w==2u) { + OX_REQUIRE(g,n->attr_countattrs[n->attr_count]); + for(i=0u;iattr_count;++i) + OX_REQUIRE(g,strcmp(n->attrs[i].name,n->attrs[n->attr_count].name)!=0,"ONNX duplicate attribute"); + ++n->attr_count; + } else if(f==7u && w==2u) { + ox_string(g,&r,name,sizeof(name)); + OX_REQUIRE(g,name[0]==0 || strcmp(name,"ai.onnx")==0,"ONNX custom domain"); + } else ox_skip(g,&r,w); + } + OX_REQUIRE(g,n->op[0]!=0 && n->output!=OX_NONE && n->count!=0u,"ONNX incomplete node"); + OX_REQUIRE(g,!g->values[n->output].defined,"ONNX duplicate producer"); g->values[n->output].defined=1u; +} +static void ox_shape_proto(ox_graph* g,ox_reader r,lw_runtime_tensor* t) { + unsigned f,w; + while(ox_tag(g,&r,&f,&w)) { + if(f==1u && w==2u) { + ox_reader d=ox_child(g,&r); unsigned df,dw; int32_t v=-1; + OX_REQUIRE(g,t->rank<8u,"ONNX input rank budget"); + while(ox_tag(g,&d,&df,&dw)) { + if(df==1u && dw==0u) { + uint64_t x=ox_var(g,&d); + OX_REQUIRE(g,x<=INT32_MAX,"ONNX input dimension overflow"); v=x==0u?-1:(int32_t)x; + } else ox_skip(g,&d,dw); + } + t->dimensions[t->rank++]=v; + } else ox_skip(g,&r,w); + } +} +static uint32_t ox_value_info(ox_graph* g,ox_reader r,int input) { + unsigned f,w; char name[4097]={0}; lw_runtime_tensor t; uint32_t id; + memset(&t,0,sizeof(t)); + while(ox_tag(g,&r,&f,&w)) { + if(f==1u && w==2u) ox_string(g,&r,name,sizeof(name)); + else if(f==2u && w==2u) { + ox_reader type=ox_child(g,&r); unsigned tf,tw; + while(ox_tag(g,&type,&tf,&tw)) { + if(tf==1u && tw==2u) { + ox_reader tensor=ox_child(g,&type); unsigned vf,vw; + while(ox_tag(g,&tensor,&vf,&vw)) { + if(vf==1u && vw==0u) { + uint64_t dtype=ox_var(g,&tensor); + OX_REQUIRE(g,dtype<=UINT32_MAX,"ONNX ValueInfo dtype overflow"); + t.dtype=ox_dtype(g,(uint32_t)dtype); + } + else if(vf==2u && vw==2u) ox_shape_proto(g,ox_child(g,&tensor),&t); + else ox_skip(g,&tensor,vw); + } + } else ox_skip(g,&type,tw); + } + } else ox_skip(g,&r,w); + } + id=ox_value_id(g,name); + if(input) { + OX_REQUIRE(g,t.dtype==LW_DTYPE_F32 && t.rank==4u && t.dimensions[1]==3 && + (t.dimensions[0]==-1 || t.dimensions[0]==1),"ONNX requires batch-one NCHW RGB input"); + OX_REQUIRE(g,!g->values[id].defined,"ONNX initializer-as-input unsupported"); + g->values[id].defined=1u; g->values[id].shape=t; + } else { + OX_REQUIRE(g,t.dtype!=0u,"ONNX output type is required"); + g->declared_output=t; + } + return id; +} +static void ox_parse_graph(ox_graph* g,ox_reader r) { + unsigned f,w; + while(ox_tag(g,&r,&f,&w)) { + if(f==1u && w==2u) ox_parse_node(g,ox_child(g,&r)); + else if(f==5u && w==2u) ox_tensor(g,ox_child(g,&r)); + else if(f==11u && w==2u) { + OX_REQUIRE(g,g->input==OX_NONE,"ONNX multiple inputs unsupported"); g->input=ox_value_info(g,ox_child(g,&r),1); + } else if(f==12u && w==2u) { + OX_REQUIRE(g,g->output==OX_NONE,"ONNX multiple outputs unsupported"); g->output=ox_value_info(g,ox_child(g,&r),0); + } else if(f==15u) ox_bad(g,LW_STATUS_UNSUPPORTED,"ONNX sparse initializers unsupported"); + else ox_skip(g,&r,w); + } +} +static ox_attr* ox_attr_find(ox_node* n,const char* name) { + uint32_t i; for(i=0u;iattr_count;++i) if(strcmp(n->attrs[i].name,name)==0) return &n->attrs[i]; + return NULL; +} +static int32_t ox_int(ox_graph* g,ox_node* n,const char* name,int32_t fallback) { + ox_attr* a=ox_attr_find(n,name); + if(a==NULL) return fallback; + OX_REQUIRE(g,a->type==2u && a->i>=INT32_MIN && a->i<=INT32_MAX,"ONNX integer attribute invalid"); + return (int32_t)a->i; +} +static void ox_list(ox_graph* g,ox_node* n,const char* name,uint8_t* p,uint32_t count,int32_t fallback) { + ox_attr* a=ox_attr_find(n,name); uint32_t i; + OX_REQUIRE(g,a==NULL || (a->type==7u && a->count==count),"ONNX list attribute length invalid"); + for(i=0u;iints[i]; + OX_REQUIRE(g,x>=INT32_MIN && x<=INT32_MAX,"ONNX list attribute overflow"); + ox_put32(p+4u*i,(uint32_t)(int32_t)x); + } +} +static float ox_fattr(ox_graph* g,ox_node* n,const char* name,float fallback) { + ox_attr* a=ox_attr_find(n,name); + OX_REQUIRE(g,a==NULL || a->type==1u,"ONNX float attribute invalid"); + return a==NULL?fallback:a->f; +} +static void ox_axes(ox_graph* g,ox_node* n,const char* name,uint32_t offset) { + ox_attr* a=ox_attr_find(n,name); uint32_t count=a==NULL?0u:a->count; + ox_put16(n->params+2u,(uint16_t)count); + if(count!=0u) ox_list(g,n,name,n->params+offset,count,0); +} +static uint16_t ox_opcode(const char* name) { + static const char* const ops[]={"","Conv","Add","Mul","Div","Erf","HardSigmoid", + "BatchNormalization","ReduceMean","Relu","AveragePool","Squeeze","Transpose", + "Unsqueeze","MatMul","Softmax","Reshape","Concat","ConvTranspose","MaxPool", + "Resize","Sigmoid","Sub","Sqrt","Pow","Slice"}; + uint16_t i; for(i=1u;i<26u;++i) if(strcmp(name,ops[i])==0) return i; + if(strcmp(name,"GlobalAveragePool")==0) return 8u; + return 0u; +} +static void ox_slice_params(ox_graph* g,ox_node* n,ox_value** in) { + uint32_t count,i,j; + if(n->count==1u) { + ox_attr* a=ox_attr_find(n,"starts"); + OX_REQUIRE(g,a!=NULL && a->count!=0u && ox_attr_find(n,"ends")!=NULL,"ONNX Slice controls missing"); + count=a->count; + ox_list(g,n,"starts",n->params+4u,count,0); ox_list(g,n,"ends",n->params+36u,count,0); + ox_list(g,n,"steps",n->params+100u,count,1); + if(ox_attr_find(n,"axes")!=NULL) ox_list(g,n,"axes",n->params+68u,count,0); + else for(i=0u;iparams+68u+i*4u,i); + } else { + OX_REQUIRE(g,n->count>=3u && in[1]->metadata && in[2]->metadata,"ONNX Slice constant controls"); + count=in[1]->control_count; + OX_REQUIRE(g,count!=0u && count==in[2]->control_count,"ONNX Slice controls mismatch"); + for(j=0u;j<4u;++j) for(i=0u;icount) { + OX_REQUIRE(g,in[j+1u]->metadata && in[j+1u]->control_count==count,"ONNX Slice controls invalid"); + x=in[j+1u]->controls[i]; + } + if(x>INT32_MAX) x=INT32_MAX; + if(xparams+4u+j*32u+i*4u,(uint32_t)(int32_t)x); + } + } + ox_put16(n->params+2u,(uint16_t)count); n->param_size=136u; n->lowered_count=1u; +} +static void ox_params(ox_graph* g,ox_node* n,ox_value** in) { + uint16_t op=n->opcode; uint8_t* p=n->params; + memset(p,0,sizeof(n->params)); n->param_size=0u; n->lowered_count=n->count; ox_put16(p,1u); + if(op==1u || op==18u || op==10u || op==19u) { + ox_attr* pad=ox_attr_find(n,"auto_pad"); uint32_t j; + n->param_size=64u; + OX_REQUIRE(g,ox_attr_find(n,"kernel_shape")!=NULL,"ONNX kernel_shape required"); + ox_list(g,n,"kernel_shape",p+8u,2u,0); ox_list(g,n,"strides",p+16u,2u,1); + if(op==1u || op==18u) { + ox_put16(p+2u,2u); ox_put32(p+4u,(uint32_t)ox_int(g,n,"group",1)); + ox_list(g,n,"dilations",p+24u,2u,1); ox_list(g,n,"pads",p+32u,4u,0); + if(op==18u) { + ox_attr* a=ox_attr_find(n,"output_padding"); + OX_REQUIRE(g,ox_attr_find(n,"output_shape")==NULL,"ONNX ConvTranspose output_shape unsupported"); + OX_REQUIRE(g,a==NULL || (a->count==2u && a->ints[0]==0 && a->ints[1]==0), + "ONNX ConvTranspose output_padding unsupported"); + } + } else { + ox_attr* a=ox_attr_find(n,"dilations"); + ox_put32(p+4u,2u); ox_list(g,n,"pads",p+24u,4u,0); + ox_put32(p+40u,(uint32_t)ox_int(g,n,"ceil_mode",0)); + ox_put32(p+44u,(uint32_t)ox_int(g,n,"count_include_pad",0)); + OX_REQUIRE(g,a==NULL || (a->count==2u && a->ints[0]==1 && a->ints[1]==1), + "ONNX pool dilation unsupported"); + } + if(pad!=NULL && strcmp(pad->string,"NOTSET")!=0) { + OX_REQUIRE(g,pad->type==3u && strcmp(pad->string,"SAME_UPPER")==0 && + lwm_read_i32(p+16u)==1 && lwm_read_i32(p+20u)==1 && + op!=18u,"ONNX dynamic auto_pad unsupported"); + for(j=0u;j<2u;++j) { + int64_t total=(int64_t)(lwm_read_i32(p+8u+j*4u)-1)* + (op==1u?lwm_read_i32(p+24u+j*4u):1); + uint32_t off=op==1u?32u:24u; + OX_REQUIRE(g,total>=0 && total<=INT32_MAX,"ONNX padding overflow"); + ox_put32(p+off+j*4u,(uint32_t)(total/2)); + ox_put32(p+off+8u+j*4u,(uint32_t)(total-total/2)); + } + } + } else if(op==6u) { + n->param_size=16u; ox_float(p+4u,ox_fattr(g,n,"alpha",0.2f)); ox_float(p+8u,ox_fattr(g,n,"beta",0.5f)); + } else if(op==7u) { + n->param_size=24u; ox_float(p+4u,ox_fattr(g,n,"epsilon",1e-5f)); ox_float(p+8u,ox_fattr(g,n,"momentum",0.9f)); + ox_put32(p+12u,(uint32_t)ox_int(g,n,"training_mode",0)); + } else if(op==8u) { + n->param_size=48u; + if(strcmp(n->op,"GlobalAveragePool")==0) { + ox_put16(p+2u,2u); ox_put32(p+4u,1u); ox_put32(p+12u,2u); ox_put32(p+16u,3u); + } else { + ox_axes(g,n,"axes",12u); ox_put32(p+4u,(uint32_t)ox_int(g,n,"keepdims",1)); + ox_put32(p+8u,(uint32_t)ox_int(g,n,"noop_with_empty_axes",0)); + } + } else if(op==11u || op==12u || op==13u) { + n->param_size=40u; ox_axes(g,n,op==12u?"perm":"axes",4u); + } else if(op==15u || op==17u) { + n->param_size=16u; ox_put32(p+4u,(uint32_t)ox_int(g,n,"axis",1)); + } else if(op==20u) { + ox_attr* mode=ox_attr_find(n,"mode"); + ox_attr* coord=ox_attr_find(n,"coordinate_transformation_mode"); + ox_attr* nearest=ox_attr_find(n,"nearest_mode"); uint32_t j; + OX_REQUIRE(g,n->count==3u && in[1]->constant && in[1]->bytes==0u && + in[2]->constant && in[2]->shape.dtype==LW_DTYPE_F32 && in[2]->bytes==16u && + mode!=NULL && strcmp(mode->string,"nearest")==0 && + coord!=NULL && strcmp(coord->string,"asymmetric")==0 && + nearest!=NULL && strcmp(nearest->string,"floor")==0,"ONNX Resize pattern unsupported"); + n->param_size=32u; n->lowered_count=1u; ox_put16(p+2u,4u); + for(j=0u;j<4u;++j) memcpy(p+4u+j*4u,in[2]->raw+j*4u,4u); + } else if(op==25u) ox_slice_params(g,n,in); + else if(op==16u) n->lowered_count=1u; +} +static void ox_numeric_shape(ox_graph* g,ox_node* n,ox_value** in,ox_value* out) { + uint8_t storage[144]={0},node[72]={0}; + lw_runtime_tensor tensors[9]; lw_model model; lw_session session; + uint32_t rank=in[0]->shape.rank,i; + memset(&model,0,sizeof(model)); memset(&session,0,sizeof(session)); memset(tensors,0,sizeof(tensors)); + for(i=0u;ilowered_count;++i) tensors[i]=in[i]->shape; + if(n->opcode==2u || n->opcode==3u || n->opcode==4u || n->opcode==14u || n->opcode==22u || n->opcode==24u) + rank=rank>in[1]->shape.rank?rank:in[1]->shape.rank; + if(n->opcode==11u) { + uint32_t axes=lwm_read_u16(n->params+2u); + if(axes!=0u) { OX_REQUIRE(g,axes<=rank,"ONNX Squeeze rank"); rank-=axes; } + else { rank=0u; for(i=0u;ishape.rank;++i) if(in[0]->shape.dimensions[i]!=1) ++rank; } + } else if(n->opcode==13u) rank+=lwm_read_u16(n->params+2u); + else if(n->opcode==8u && lwm_read_u32(n->params+4u)==0u) { + uint32_t axes=lwm_read_u16(n->params+2u); + OX_REQUIRE(g,axes<=rank,"ONNX ReduceMean rank"); rank=axes==0u?0u:rank-axes; + } else if(n->opcode==16u) { + uint64_t elements=1u,known=1u; int32_t missing=-1; + OX_REQUIRE(g,n->count==2u && in[1]->metadata && in[1]->control_count!=0u,"ONNX Reshape controls"); + rank=in[1]->control_count; + for(i=0u;ishape.rank;++i) { + OX_REQUIRE(g,elements<=UINT64_MAX/(uint32_t)in[0]->shape.dimensions[i],"ONNX Reshape overflow"); + elements*=(uint32_t)in[0]->shape.dimensions[i]; + } + for(i=0u;icontrols[i]; + if(d==0) { OX_REQUIRE(g,ishape.rank,"ONNX Reshape copy axis"); d=in[0]->shape.dimensions[i]; } + OX_REQUIRE(g,d>=-1 && d!=0 && d<=INT32_MAX,"ONNX Reshape dimension"); + tensors[8].dimensions[i]=(int32_t)d; + if(d==-1) { OX_REQUIRE(g,missing==-1,"ONNX Reshape multiple -1"); missing=(int32_t)i; } + else { OX_REQUIRE(g,known<=UINT64_MAX/(uint32_t)d,"ONNX Reshape overflow"); known*=(uint32_t)d; } + } + if(missing>=0) { + OX_REQUIRE(g,elements%known==0u && elements/known<=INT32_MAX,"ONNX Reshape mismatch"); + tensors[8].dimensions[missing]=(int32_t)(elements/known); + } + } + OX_REQUIRE(g,rank<=8u,"ONNX resolved rank budget"); + tensors[8].dtype=in[0]->shape.dtype; tensors[8].rank=rank; + if(n->opcode!=16u) for(i=0u;iopcode); ox_put16(node+2u,(uint16_t)n->lowered_count); ox_put16(node+4u,1u); + for(i=0u;ilowered_count;++i) ox_put32(node+8u+i*4u,i); + ox_put32(node+40u,8u); + if(n->param_size!=0u) { memcpy(storage+8u,n->params,n->param_size); ox_put64(node+56u,8u); } + g->status=lw_resolve_import_node(&session,node,OX_MAX_BYTES,g->error); + if(g->status!=LW_STATUS_OK) longjmp(*g->escape,1); + out->shape=tensors[8]; out->ready=1u; out->metadata=0u; +} +static void ox_metadata(ox_graph* g,ox_node* n,ox_value** in,ox_value* out) { + uint32_t i,j,count=0u; out->metadata=1u; out->ready=1u; + if(strcmp(n->op,"Shape")==0) { + out->control_count=in[0]->shape.rank; + for(i=0u;icontrol_count;++i) out->controls[i]=in[0]->shape.dimensions[i]; + out->shape.dtype=LW_DTYPE_I64; out->shape.rank=1u; + out->shape.dimensions[0]=(int32_t)out->control_count; + } else if(strcmp(n->op,"Concat")==0) { + OX_REQUIRE(g,ox_int(g,n,"axis",0)==0,"ONNX metadata Concat axis"); + for(i=0u;icount;++i) { + OX_REQUIRE(g,in[i]->metadata && in[i]->control_count<=8u-count,"ONNX metadata Concat budget"); + for(j=0u;jcontrol_count;++j) out->controls[count++]=in[i]->controls[j]; + } + out->control_count=count; + } else if(strcmp(n->op,"Slice")==0) { + int64_t start,end,step; int32_t len=(int32_t)in[0]->control_count; + OX_REQUIRE(g,lwm_read_u16(n->params+2u)==1u && lwm_read_i32(n->params+68u)==0,"ONNX metadata Slice axes"); + start=lwm_read_i32(n->params+4u); end=lwm_read_i32(n->params+36u); step=lwm_read_i32(n->params+100u); + OX_REQUIRE(g,step>0,"ONNX metadata Slice step"); + if(start<0) start+=len; + if(end<0) end+=len; + if(start<0) start=0; + if(start>len) start=len; + if(end<0) end=0; + if(end>len) end=len; + for(;startcontrols[count++]=in[0]->controls[start]; + out->control_count=count; + } else if(strcmp(n->op,"Squeeze")==0 || strcmp(n->op,"Unsqueeze")==0) { + out->control_count=in[0]->control_count; memcpy(out->controls,in[0]->controls,sizeof(out->controls)); + } else ox_bad(g,LW_STATUS_UNSUPPORTED,"ONNX metadata reaches unsupported operator"); + if(strcmp(n->op,"Shape")!=0) { + ox_numeric_shape(g,n,in,out); + out->metadata=1u; + } +} +static void ox_check_attributes(ox_graph* g,ox_node* n) { + const char* allowed=","; + uint32_t i; + switch(n->opcode) { + case 1u: allowed=",auto_pad,kernel_shape,strides,dilations,pads,group,"; break; + case 18u: allowed=",auto_pad,kernel_shape,strides,dilations,pads,group,output_padding,output_shape,"; break; + case 6u: allowed=",alpha,beta,"; break; + case 7u: allowed=",epsilon,momentum,training_mode,spatial,"; break; + case 8u: allowed=strcmp(n->op,"GlobalAveragePool")==0?",":",axes,keepdims,noop_with_empty_axes,"; break; + case 10u: case 19u: allowed=",auto_pad,kernel_shape,strides,pads,dilations,ceil_mode,count_include_pad,storage_order,"; break; + case 11u: case 13u: allowed=",axes,"; break; + case 12u: allowed=",perm,"; break; + case 15u: case 17u: allowed=",axis,"; break; + case 20u: allowed=",mode,coordinate_transformation_mode,nearest_mode,"; break; + case 25u: allowed=",starts,ends,axes,steps,"; break; + default: break; + } + for(i=0u;iattr_count;++i) { + char key[68]; (void)snprintf(key,sizeof(key),",%s,",n->attrs[i].name); + OX_REQUIRE(g,strstr(allowed,key)!=NULL,"ONNX unsupported operator attribute"); + } + if(n->opcode==7u) OX_REQUIRE(g,ox_int(g,n,"spatial",1)==1,"ONNX nonspatial BN unsupported"); + if(n->opcode==19u) OX_REQUIRE(g,ox_int(g,n,"storage_order",0)==0,"ONNX pool storage order unsupported"); +} +static void ox_arity(ox_graph* g,ox_node* n) { + uint32_t lo=1u,hi=1u; + if(n->opcode==1u || n->opcode==18u) { lo=2u; hi=3u; } + else if(n->opcode==2u || n->opcode==3u || n->opcode==4u || n->opcode==14u || + n->opcode==16u || n->opcode==22u || n->opcode==24u) lo=hi=2u; + else if(n->opcode==7u) lo=hi=5u; + else if(n->opcode==20u) lo=hi=3u; + else if(n->opcode==17u) hi=8u; + else if(n->opcode==25u) hi=5u; + OX_REQUIRE(g,n->count>=lo && n->count<=hi,"ONNX operator arity invalid"); +} +/* Shape-only passes at two representative sizes discover changing axes for + * this PP-OCR subset. Runtime session creation resolves the actual size using + * the same shape rules; this is not a general symbolic ONNX shape engine. */ +static void ox_lower(ox_graph* g,unsigned pass) { + uint32_t i,j; lw_runtime_tensor input_shape=g->values[g->input].shape; + for(i=0u;ivalue_count;++i) { + ox_value* v=&g->values[i]; v->ready=v->constant; + if(!v->constant) v->metadata=0u; + } + input_shape.dimensions[0]=1; + for(i=2u;i<4u;++i) if(g->values[g->input].dynamic[i]==-1) + input_shape.dimensions[i]=pass==0u?320:640; + g->values[g->input].shape=input_shape; g->values[g->input].ready=1u; + for(i=0u;inode_count;++i) { + ox_node* n=&g->nodes[i]; ox_value* in[8]; ox_value* out=&g->values[n->output]; + if(strcmp(n->op,"Identity")==0) { + uint32_t root; + OX_REQUIRE(g,n->count==1u && n->inputs[0]!=OX_NONE && n->attr_count==0u,"ONNX Identity arity"); + root=ox_root(g,n->inputs[0]); + OX_REQUIRE(g,g->values[root].ready,"ONNX graph not topologically ordered"); + out->alias=root; out->ready=1u; n->emitted=0u; continue; + } + n->opcode=ox_opcode(n->op); ox_arity(g,n); ox_check_attributes(g,n); + for(j=0u;jcount;++j) { + OX_REQUIRE(g,n->inputs[j]!=OX_NONE,"ONNX omitted input unsupported"); + in[j]=&g->values[ox_root(g,n->inputs[j])]; + OX_REQUIRE(g,in[j]->ready,"ONNX graph not topologically ordered"); + } + if(strcmp(n->op,"Shape")==0) { + OX_REQUIRE(g,n->count==1u && n->attr_count==0u,"ONNX Shape contract"); + ox_metadata(g,n,in,out); n->emitted=0u; continue; + } + OX_REQUIRE(g,n->opcode!=0u,"ONNX operator unsupported"); ox_params(g,n,in); + if(in[0]->metadata) { ox_metadata(g,n,in,out); n->emitted=0u; continue; } + for(j=1u;jlowered_count;++j) + OX_REQUIRE(g,!in[j]->metadata,"ONNX shape controls reach numeric operator"); + ox_numeric_shape(g,n,in,out); n->emitted=1u; + if(pass==0u) out->first_rank=out->shape.rank; + OX_REQUIRE(g,out->first_rank==out->shape.rank,"ONNX varying tensor rank unsupported"); + for(j=0u;jshape.rank;++j) { + if(pass==0u) out->first[j]=out->shape.dimensions[j]; + out->dynamic[j]=pass==0u?out->first[j]: + (out->first[j]==out->shape.dimensions[j]?out->first[j]:-1); + } + if(n->opcode==16u && pass!=0u) { + uint32_t unresolved=0u; + for(j=0u;jshape.rank;++j) if(out->dynamic[j]==-1) ++unresolved; + OX_REQUIRE(g,unresolved<=1u,"ONNX dynamic Reshape multiple varying axes"); + } + } +} +static size_t ox_align(size_t n) { return (n+7u)&~(size_t)7u; } +static void ox_serialize(ox_graph* g,lw_model* model) { + uint32_t i,j,tensors=0u,nodes=0u; + size_t tensor_off,node_off,param_off,param_bytes=0u,weight_off,weight_bytes=0u,size,k; + uint8_t* b; uint64_t hash=UINT64_C(14695981039346656037); + g->output=ox_root(g,g->output); + OX_REQUIRE(g,g->values[g->output].ready && !g->values[g->output].metadata,"ONNX output not numeric"); + OX_REQUIRE(g,g->declared_output.dtype==g->values[g->output].shape.dtype && + g->declared_output.rank==g->values[g->output].shape.rank, + "ONNX output type/rank does not match graph"); + for(i=0u;ideclared_output.rank;++i) + OX_REQUIRE(g,g->declared_output.dimensions[i]<=0 || + g->declared_output.dimensions[i]==g->values[g->output].shape.dimensions[i], + "ONNX output dimension does not match graph"); + g->values[g->input].used=1u; g->values[g->output].used=1u; + for(i=0u;inode_count;++i) if(g->nodes[i].emitted) { + ox_node* n=&g->nodes[i]; ++nodes; param_bytes+=ox_align(n->param_size); + g->values[n->output].used=1u; + for(j=0u;jlowered_count;++j) g->values[ox_root(g,n->inputs[j])].used=1u; + } + for(i=0u;ivalue_count;++i) if(g->values[i].used) { + g->values[i].index=tensors++; + if(g->values[i].constant) weight_bytes+=ox_align(g->values[i].bytes); + } + tensor_off=176u; node_off=tensor_off+(size_t)tensors*80u; + param_off=node_off+(size_t)nodes*72u; weight_off=param_off+param_bytes; size=weight_off+weight_bytes; + OX_REQUIRE(g,size<=OX_MAX_BYTES,"ONNX normalized graph budget"); + b=(uint8_t*)calloc(size,1u); + if(b==NULL) ox_bad(g,LW_STATUS_OUT_OF_MEMORY,"ONNX normalized graph allocation"); + memcpy(b,"LWM0",4u); ox_put16(b+6u,1u); ox_put32(b+8u,160u); ox_put32(b+12u,1u); + ox_put32(b+16u,tensors); ox_put32(b+20u,nodes); ox_put32(b+24u,1u); ox_put32(b+28u,1u); + ox_put64(b+32u,160u); ox_put64(b+40u,168u); ox_put64(b+48u,tensor_off); + ox_put64(b+56u,node_off); ox_put64(b+64u,param_off); ox_put64(b+72u,param_bytes); + ox_put64(b+80u,weight_off); ox_put64(b+96u,weight_off); ox_put64(b+104u,weight_bytes); + ox_put64(b+112u,size); + ox_put32(b+160u,g->values[g->input].index); ox_put32(b+168u,g->values[g->output].index); + weight_bytes=0u; + for(i=0u;ivalue_count;++i) if(g->values[i].used) { + ox_value* v=&g->values[i]; uint8_t* t=b+tensor_off+v->index*80u; + ox_put32(t,v->shape.dtype); ox_put32(t+4u,v->shape.rank); + for(j=0u;jshape.rank;++j) + ox_put32(t+8u+j*4u,(uint32_t)(v->constant?v->shape.dimensions[j]:v->dynamic[j])); + ox_put32(t+40u,(v->constant?1u:0u)|(i==g->input?2u:0u)|(i==g->output?4u:0u)); ox_put64(t+64u,UINT64_MAX); + if(v->constant && v->bytes!=0u) { + ox_put64(t+48u,weight_off+weight_bytes); ox_put64(t+56u,v->bytes); + memcpy(b+weight_off+weight_bytes,v->raw,v->bytes); weight_bytes+=ox_align(v->bytes); + } + } + nodes=0u; param_bytes=0u; + for(i=0u;inode_count;++i) if(g->nodes[i].emitted) { + ox_node* n=&g->nodes[i]; uint8_t* p=b+node_off+nodes++*72u; + ox_put16(p,n->opcode); ox_put16(p+2u,(uint16_t)n->lowered_count); ox_put16(p+4u,1u); + for(j=0u;jlowered_count;++j) ox_put32(p+8u+j*4u,g->values[ox_root(g,n->inputs[j])].index); + ox_put32(p+40u,g->values[n->output].index); + if(n->param_size!=0u) { + ox_put64(p+56u,param_off+param_bytes); ox_put32(p+64u,n->param_size); + memcpy(b+param_off+param_bytes,n->params,n->param_size); param_bytes+=ox_align(n->param_size); + } + } + for(k=0u;kbytes); model->bytes=b; model->byte_count=size; +} +lw_status lw_import_onnx(lw_model* model,lw_error* error) { + ox_graph* g=(ox_graph*)calloc(1u,sizeof(*g)); lw_status status; uint32_t i; + if(g==NULL) { lw_set_error(error,LW_STATUS_OUT_OF_MEMORY,"ONNX importer allocation"); return LW_STATUS_OUT_OF_MEMORY; } + g->error=error; g->input=OX_NONE; g->output=OX_NONE; g->status=LW_STATUS_INVALID_FORMAT; + g->escape=(jmp_buf*)malloc(sizeof(*g->escape)); + if(g->escape==NULL) { + free(g); lw_set_error(error,LW_STATUS_OUT_OF_MEMORY,"ONNX error context allocation"); + return LW_STATUS_OUT_OF_MEMORY; + } + if(setjmp(*g->escape)==0) { + ox_reader r={model->bytes,model->bytes+model->byte_count}; unsigned f,w,graphs=0u; + OX_REQUIRE(g,model->byte_count<=OX_MAX_BYTES,"ONNX file budget exceeded"); + while(ox_tag(g,&r,&f,&w)) { + if(f==7u && w==2u) { + ox_reader graph=ox_child(g,&r),scan=graph; + unsigned sf,sw; uint32_t constants=0u,nodes=0u; + OX_REQUIRE(g,++graphs==1u,"ONNX duplicate graph"); + while(ox_tag(g,&scan,&sf,&sw)) { + if(sf==1u) ++nodes; + if(sf==5u) ++constants; + OX_REQUIRE(g,nodes<=OX_MAX_NODES && constants<=OX_MAX_VALUES,"ONNX table budget"); + ox_skip(g,&scan,sw); + } + OX_REQUIRE(g,nodes!=0u,"ONNX empty graph unsupported"); + g->node_capacity=nodes; + g->value_capacity=nodes*9u+constants+2u; + if(g->value_capacity>OX_MAX_VALUES) g->value_capacity=OX_MAX_VALUES; + g->values=(ox_value*)calloc(g->value_capacity,sizeof(*g->values)); + g->nodes=(ox_node*)calloc(g->node_capacity,sizeof(*g->nodes)); + if(g->values==NULL || g->nodes==NULL) ox_bad(g,LW_STATUS_OUT_OF_MEMORY,"ONNX graph tables allocation"); + ox_parse_graph(g,graph); + } else if(f==8u && w==2u) { + ox_reader c=ox_child(g,&r); unsigned cf,cw; char domain[128]={0}; uint64_t version=0u; + while(ox_tag(g,&c,&cf,&cw)) { + if(cf==1u && cw==2u) ox_string(g,&c,domain,sizeof(domain)); + else if(cf==2u && cw==0u) version=ox_var(g,&c); + else ox_skip(g,&c,cw); + } + OX_REQUIRE(g,(domain[0]==0 || strcmp(domain,"ai.onnx")==0) && g->opset==0u && + (version==7u || version==11u || version==14u),"ONNX standard opset 7/11/14 required"); + g->opset=(uint32_t)version; + } else if(f==25u) ox_bad(g,LW_STATUS_UNSUPPORTED,"ONNX functions unsupported"); + else ox_skip(g,&r,w); + } + OX_REQUIRE(g,graphs==1u && g->opset!=0u && g->input!=OX_NONE && g->output!=OX_NONE,"ONNX missing graph contract"); + memcpy(g->values[g->input].dynamic,g->values[g->input].shape.dimensions,32u); + g->values[g->input].dynamic[0]=1; + ox_lower(g,0u); ox_lower(g,1u); ox_serialize(g,model); + g->status=lw_validate_lwm_v0(model,error); + } + status=g->status; + if(g->values!=NULL) for(i=0u;ivalue_count;++i) { free(g->values[i].name); free(g->values[i].owned); } + free(g->values); free(g->nodes); free(g->escape); free(g); return status; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/operator_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/operator_internal.h new file mode 100644 index 00000000..b8535191 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/operator_internal.h @@ -0,0 +1,37 @@ +#ifndef LW_OPERATOR_INTERNAL_H +#define LW_OPERATOR_INTERNAL_H + +/* + * LWM operator identifiers are an internal runtime contract. Keep the + * numeric values in one place so execution, planning, and analysis cannot + * silently drift apart. This header is private and is not part of the C ABI. + */ +enum { + LW_OP_CONV = 1, + LW_OP_ADD = 2, + LW_OP_MUL = 3, + LW_OP_DIV = 4, + LW_OP_ERF = 5, + LW_OP_HARD_SIGMOID = 6, + LW_OP_BATCH_NORMALIZATION = 7, + LW_OP_REDUCE_MEAN = 8, + LW_OP_RELU = 9, + LW_OP_AVERAGE_POOL = 10, + LW_OP_SQUEEZE = 11, + LW_OP_TRANSPOSE = 12, + LW_OP_UNSQUEEZE = 13, + LW_OP_MATMUL = 14, + LW_OP_SOFTMAX = 15, + LW_OP_RESHAPE = 16, + LW_OP_CONCAT = 17, + LW_OP_CONV_TRANSPOSE = 18, + LW_OP_MAX_POOL = 19, + LW_OP_RESIZE = 20, + LW_OP_SIGMOID = 21, + LW_OP_SUB = 22, + LW_OP_SQRT = 23, + LW_OP_POW = 24, + LW_OP_SLICE = 25 +}; + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/parallel.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/parallel.c new file mode 100644 index 00000000..2ce6c741 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/parallel.c @@ -0,0 +1,262 @@ +#include "parallel_internal.h" + +#include +#include + +#if defined(_WIN32) +# define WIN32_LEAN_AND_MEAN +# include +# include +#elif !defined(__EMSCRIPTEN__) +# include +#endif + +typedef struct lw_pool_worker { + struct lw_thread_pool* pool; + uint32_t worker_index; +} lw_pool_worker; + +struct lw_thread_pool { + uint32_t worker_count; + uint32_t active_worker_count; + uint32_t pending_worker_count; + uint64_t generation; + uint32_t stopping; + lw_parallel_callback callback; + void* context; + lw_pool_worker workers[LW_PARALLEL_MAX_WORKERS - 1u]; +#if defined(_WIN32) + CRITICAL_SECTION mutex; + CONDITION_VARIABLE work_condition; + CONDITION_VARIABLE done_condition; + HANDLE threads[LW_PARALLEL_MAX_WORKERS - 1u]; +#elif !defined(__EMSCRIPTEN__) + pthread_mutex_t mutex; + pthread_cond_t work_condition; + pthread_cond_t done_condition; + pthread_t threads[LW_PARALLEL_MAX_WORKERS - 1u]; +#endif +}; + +#if defined(_WIN32) +static void lock_pool(lw_thread_pool* pool) { + EnterCriticalSection(&pool->mutex); +} +static void unlock_pool(lw_thread_pool* pool) { + LeaveCriticalSection(&pool->mutex); +} +static void wait_for_work(lw_thread_pool* pool) { + (void)SleepConditionVariableCS(&pool->work_condition, &pool->mutex, INFINITE); +} +static void wait_for_done(lw_thread_pool* pool) { + (void)SleepConditionVariableCS(&pool->done_condition, &pool->mutex, INFINITE); +} +static void wake_all_workers(lw_thread_pool* pool) { + WakeAllConditionVariable(&pool->work_condition); +} +static void wake_caller(lw_thread_pool* pool) { + WakeConditionVariable(&pool->done_condition); +} +#elif !defined(__EMSCRIPTEN__) +static void lock_pool(lw_thread_pool* pool) { + (void)pthread_mutex_lock(&pool->mutex); +} +static void unlock_pool(lw_thread_pool* pool) { + (void)pthread_mutex_unlock(&pool->mutex); +} +static void wait_for_work(lw_thread_pool* pool) { + (void)pthread_cond_wait(&pool->work_condition, &pool->mutex); +} +static void wait_for_done(lw_thread_pool* pool) { + (void)pthread_cond_wait(&pool->done_condition, &pool->mutex); +} +static void wake_all_workers(lw_thread_pool* pool) { + (void)pthread_cond_broadcast(&pool->work_condition); +} +static void wake_caller(lw_thread_pool* pool) { + (void)pthread_cond_signal(&pool->done_condition); +} +#endif + +#if !defined(__EMSCRIPTEN__) +static void worker_loop(lw_pool_worker* worker) { + lw_thread_pool* pool = worker->pool; + uint64_t observed_generation = 0u; + for (;;) { + lw_parallel_callback callback; + void* context; + uint32_t active_worker_count; + lock_pool(pool); + while (pool->stopping == 0u && pool->generation == observed_generation) { + wait_for_work(pool); + } + if (pool->stopping != 0u) { + unlock_pool(pool); + return; + } + observed_generation = pool->generation; + callback = pool->callback; + context = pool->context; + active_worker_count = pool->active_worker_count; + unlock_pool(pool); + if (worker->worker_index >= active_worker_count) { + continue; + } + callback(context, worker->worker_index, active_worker_count); + lock_pool(pool); + --pool->pending_worker_count; + if (pool->pending_worker_count == 0u) { + wake_caller(pool); + } + unlock_pool(pool); + } +} +#endif + +#if defined(_WIN32) +static unsigned __stdcall pool_worker_entry(void* context) { + worker_loop((lw_pool_worker*)context); + return 0u; +} +#elif !defined(__EMSCRIPTEN__) +static void* pool_worker_entry(void* context) { + worker_loop((lw_pool_worker*)context); + return NULL; +} +#endif + +lw_thread_pool* lw_thread_pool_create(uint32_t worker_count) { +#if defined(__EMSCRIPTEN__) + (void)worker_count; + return NULL; +#else + lw_thread_pool* pool; + uint32_t worker_index; + if (worker_count < 2u) { + return NULL; + } + if (worker_count > LW_PARALLEL_MAX_WORKERS) { + worker_count = LW_PARALLEL_MAX_WORKERS; + } + pool = (lw_thread_pool*)calloc(1u, sizeof(*pool)); + if (pool == NULL) { + return NULL; + } +#if defined(_WIN32) + InitializeCriticalSection(&pool->mutex); + InitializeConditionVariable(&pool->work_condition); + InitializeConditionVariable(&pool->done_condition); +#else + if (pthread_mutex_init(&pool->mutex, NULL) != 0) { + free(pool); + return NULL; + } + if (pthread_cond_init(&pool->work_condition, NULL) != 0) { + (void)pthread_mutex_destroy(&pool->mutex); + free(pool); + return NULL; + } + if (pthread_cond_init(&pool->done_condition, NULL) != 0) { + (void)pthread_cond_destroy(&pool->work_condition); + (void)pthread_mutex_destroy(&pool->mutex); + free(pool); + return NULL; + } +#endif + pool->worker_count = 1u; + for (worker_index = 1u; worker_index < worker_count; ++worker_index) { + lw_pool_worker* worker = &pool->workers[worker_index - 1u]; + worker->pool = pool; + worker->worker_index = worker_index; +#if defined(_WIN32) + pool->threads[worker_index - 1u] = + (HANDLE)_beginthreadex(NULL, 0u, pool_worker_entry, worker, 0u, NULL); + if (pool->threads[worker_index - 1u] == NULL) { + break; + } +#else + if (pthread_create(&pool->threads[worker_index - 1u], NULL, pool_worker_entry, worker) != 0) { + break; + } +#endif + ++pool->worker_count; + } + if (pool->worker_count < 2u) { + lw_thread_pool_free(pool); + return NULL; + } + return pool; +#endif +} + +void lw_thread_pool_free(lw_thread_pool* pool) { +#if defined(__EMSCRIPTEN__) + (void)pool; +#else + uint32_t worker_index; + if (pool == NULL) { + return; + } + lock_pool(pool); + pool->stopping = 1u; + ++pool->generation; + wake_all_workers(pool); + unlock_pool(pool); + for (worker_index = 1u; worker_index < pool->worker_count; ++worker_index) { +#if defined(_WIN32) + WaitForSingleObject(pool->threads[worker_index - 1u], INFINITE); + CloseHandle(pool->threads[worker_index - 1u]); +#else + (void)pthread_join(pool->threads[worker_index - 1u], NULL); +#endif + } +#if defined(_WIN32) + DeleteCriticalSection(&pool->mutex); +#else + (void)pthread_cond_destroy(&pool->done_condition); + (void)pthread_cond_destroy(&pool->work_condition); + (void)pthread_mutex_destroy(&pool->mutex); +#endif + free(pool); +#endif +} + +uint32_t lw_thread_pool_worker_count(const lw_thread_pool* pool) { + return pool == NULL ? 1u : pool->worker_count; +} + +void lw_thread_pool_run(lw_thread_pool* pool, uint32_t worker_count, + lw_parallel_callback callback, void* context) { +#if defined(__EMSCRIPTEN__) + (void)pool; + (void)worker_count; + if (callback != NULL) { + callback(context, 0u, 1u); + } +#else + if (callback == NULL) { + return; + } + if (pool == NULL || worker_count < 2u) { + callback(context, 0u, 1u); + return; + } + if (worker_count > pool->worker_count) { + worker_count = pool->worker_count; + } + lock_pool(pool); + pool->callback = callback; + pool->context = context; + pool->active_worker_count = worker_count; + pool->pending_worker_count = worker_count - 1u; + ++pool->generation; + wake_all_workers(pool); + unlock_pool(pool); + callback(context, 0u, worker_count); + lock_pool(pool); + while (pool->pending_worker_count != 0u) { + wait_for_done(pool); + } + unlock_pool(pool); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/parallel_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/parallel_internal.h new file mode 100644 index 00000000..1fcaba3e --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/parallel_internal.h @@ -0,0 +1,21 @@ +#ifndef LW_PARALLEL_INTERNAL_H +#define LW_PARALLEL_INTERNAL_H + +/* Small dependency-free parallel-for used by coarse inference kernels. */ + +#include + +#define LW_PARALLEL_MAX_WORKERS 16u + +typedef void (*lw_parallel_callback)(void* context, uint32_t worker_index, + uint32_t worker_count); + +typedef struct lw_thread_pool lw_thread_pool; + +lw_thread_pool* lw_thread_pool_create(uint32_t worker_count); +void lw_thread_pool_free(lw_thread_pool* pool); +uint32_t lw_thread_pool_worker_count(const lw_thread_pool* pool); +void lw_thread_pool_run(lw_thread_pool* pool, uint32_t worker_count, + lw_parallel_callback callback, void* context); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/prepared_execution.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/prepared_execution.c new file mode 100644 index 00000000..6817979d --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/prepared_execution.c @@ -0,0 +1,147 @@ +#include "session_internal.h" + +#include "error_internal.h" +#include "lwm_read.h" +#include "packed_conv_internal.h" +#include "packed_conv3x3_internal.h" +#include "../simd/simd_kernels.h" + +#include +#include + +#if defined(LW_EXPERIMENTAL_PREPARED_EXECUTION) +static lw_packed_conv3x3_kernel_fn select_conv3x3_kernel(const lw_session* session, + uint16_t* kernel_id) { + if (lw_simd_level_is_avx2(session->cpu.simd)) { + *kernel_id = LW_CONV3X3_KERNEL_AVX2; + return lw_avx2_packed_conv3x3_stride2_pad1_f32; + } + *kernel_id = LW_CONV3X3_KERNEL_SCALAR; + return lw_scalar_packed_conv3x3_stride2_pad1_f32; +} + +static lw_packed_conv1x1_kernel_fn select_conv1x1_kernel(const lw_session* session, + uint16_t* kernel_id) { + if (lw_simd_level_is_avx2(session->cpu.simd)) { + *kernel_id = LW_CONV1X1_KERNEL_AVX2; + return lw_avx2_packed_conv1x1_f32; + } + if (lw_simd_level_is_neon(session->cpu.simd)) { + *kernel_id = LW_CONV1X1_KERNEL_NEON; + return lw_neon_packed_conv1x1_f32; + } + if (lw_simd_level_is_lsx(session->cpu.simd)) { + *kernel_id = LW_CONV1X1_KERNEL_LSX; + return lw_lsx_packed_conv1x1_f32; + } + if (lw_simd_level_is_sse2(session->cpu.simd)) { + *kernel_id = LW_CONV1X1_KERNEL_SSE2; + return lw_sse2_packed_conv1x1_f32; + } + *kernel_id = LW_CONV1X1_KERNEL_SCALAR; + return lw_scalar_packed_conv1x1_f32; +} + +#endif + +void lw_free_execution_nodes(lw_session* session) { + if (session == NULL) { + return; + } + free(session->execution_nodes); + session->execution_nodes = NULL; + session->execution_node_count = 0u; +} + +lw_status lw_prepare_execution_nodes(lw_session* session, lw_error* error) { + uint32_t node_index; + + if (session == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "session is required"); + return LW_STATUS_INVALID_ARGUMENT; + } + lw_free_execution_nodes(session); +#if defined(LW_EXPERIMENTAL_PREPARED_EXECUTION) + if (session->model->info.node_count == 0u) { + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + } + if ((size_t)session->model->info.node_count > + SIZE_MAX / sizeof(*session->execution_nodes)) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, + "execution node table size overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + session->execution_nodes = (lw_bound_node*)calloc( + session->model->info.node_count, sizeof(*session->execution_nodes)); + if (session->execution_nodes == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, + "unable to allocate execution node table"); + return LW_STATUS_OUT_OF_MEMORY; + } + session->execution_node_count = session->model->info.node_count; + for (node_index = 0u; node_index < session->execution_node_count; ++node_index) { + const uint8_t* node = session->model->bytes + + (size_t)session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + lw_bound_node* bound = &session->execution_nodes[node_index]; + uint16_t input_count = lwm_read_u16(node + 2u); + uint32_t input_index; + + if (input_count > LWM_V0_MAX_NODE_INPUTS) { + lw_free_execution_nodes(session); + lw_set_error(error, LW_STATUS_INVALID_FORMAT, + "execution node input count exceeds the LWM limit"); + return LW_STATUS_INVALID_FORMAT; + } + bound->node_index = node_index; + bound->operator_type = lwm_read_u16(node); + bound->execution_kind = LW_BOUND_EXEC_GENERIC; + bound->input_count = input_count; + bound->output_index = lwm_read_u32(node + 40u); + for (input_index = 0u; input_index < input_count; ++input_index) { + bound->input_indices[input_index] = lwm_read_u32( + node + 8u + (size_t)input_index * sizeof(uint32_t)); + } + if (session->prepared_constants != NULL && + session->prepared_constants[node_index].kind != LW_PREPARED_CONSTANT_NONE) { + bound->implementation = session->prepared_constants[node_index].kind; + bound->prepared_constant = &session->prepared_constants[node_index]; + if (bound->operator_type == 1u && + bound->implementation == LW_PREPARED_CONSTANT_CONV1X1_PACKED4 && + input_count >= 2u) { + bound->execution_kind = LW_BOUND_EXEC_CONV1X1_PACKED; + bound->data.conv1x1.kernel = select_conv1x1_kernel( + session, &bound->data.conv1x1.kernel_id); + bound->data.conv1x1.packed_weights = + (const float*)(const void*)(session->packed_weights + + (size_t)session->prepared_constants[node_index].packed_weight_offset); + bound->data.conv1x1.input_index = bound->input_indices[0]; + bound->data.conv1x1.bias_index = + input_count == 3u ? bound->input_indices[2] : UINT32_MAX; + bound->data.conv1x1.output_index = bound->output_index; + bound->data.conv1x1.output_tile = LW_PACKED_CONV1X1_OUTPUT_TILE; + } else if (bound->operator_type == 1u && + bound->implementation == LW_PREPARED_CONSTANT_CONV3X3_STRIDE2_PACKED8 && + input_count >= 2u) { + bound->execution_kind = LW_BOUND_EXEC_CONV3X3_PACKED; + bound->data.conv3x3.kernel = select_conv3x3_kernel( + session, &bound->data.conv3x3.kernel_id); + bound->data.conv3x3.packed_weights = + (const float*)(const void*)(session->packed_weights + + (size_t)session->prepared_constants[node_index].packed_weight_offset); + bound->data.conv3x3.input_index = bound->input_indices[0]; + bound->data.conv3x3.bias_index = + input_count == 3u ? bound->input_indices[2] : UINT32_MAX; + bound->data.conv3x3.output_index = bound->output_index; + bound->data.conv3x3.output_tile = + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + } + } + } +#else + (void)node_index; +#endif + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/rec_backend_kernels.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/rec_backend_kernels.c new file mode 100644 index 00000000..6b2b6a5d --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/rec_backend_kernels.c @@ -0,0 +1,58 @@ +#include "rec_backend_kernels_internal.h" +#include "../simd/simd_kernels.h" + +#if defined(__EMSCRIPTEN__) +void lw_wasm128_nhwc_pointwise_2x16_f32(const float*, const float*, + const lw_nhwc_epilogue*, float*, + uint32_t, uint32_t, uint32_t); +void lw_wasm128_nhwc_pointwise_4x16_f32(const float*, const float*, + const lw_nhwc_epilogue*, float*, + uint32_t, uint32_t, uint32_t); +#ifndef LW_WASM_POINTWISE_ROWS +#define LW_WASM_POINTWISE_ROWS 2 +#endif +#if LW_WASM_POINTWISE_ROWS == 2 +#define LW_WASM_POINTWISE_KERNEL lw_wasm128_nhwc_pointwise_2x16_f32 +#elif LW_WASM_POINTWISE_ROWS == 4 +#define LW_WASM_POINTWISE_KERNEL lw_wasm128_nhwc_pointwise_4x16_f32 +#else +#error Unsupported LW_WASM_POINTWISE_ROWS +#endif +void lw_wasm128_packed_matmul_argmax_scores_f32( + const float*, const float*, const float*, uint32_t*, float*, + uint32_t, uint32_t, uint32_t); +void lw_wasm128_ctc_row_probabilities_f32( + const float*, const float*, const float*, const uint32_t*, const float*, + float*, uint32_t, uint32_t, uint32_t); +#else +static void x64_ctc_argmax_scores(const float* input, const float* weights, + const float* bias, uint32_t* indices, + float* scores, uint32_t rows, + uint32_t inner, uint32_t columns) { + lw_avx2_fma_packed_matmul_argmax_scores_f32( + input, weights, bias, indices, scores, 1u, rows, inner, columns); +} +#endif + +const lw_rec_backend_kernels* lw_rec_backend_kernels_current(void) { +#if defined(__EMSCRIPTEN__) + static const lw_rec_backend_kernels kernels = { + "wasm128", 4u, LW_NHWC_OC_BLOCK, 0u, + LW_WASM_POINTWISE_KERNEL, + lw_avx2_fma_nhwc_dense_f32, + lw_avx2_fma_nhwc_depthwise_f32, + lw_wasm128_packed_matmul_argmax_scores_f32, + lw_wasm128_ctc_row_probabilities_f32 + }; +#else + static const lw_rec_backend_kernels kernels = { + "avx2-fma", 8u, LW_NHWC_OC_BLOCK, 1u, + lw_avx2_fma_nhwc_pointwise_f32, + lw_avx2_fma_nhwc_dense_f32, + lw_avx2_fma_nhwc_depthwise_f32, + x64_ctc_argmax_scores, + lw_avx2_ctc_row_probabilities_f32 + }; +#endif + return &kernels; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/rec_backend_kernels_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/rec_backend_kernels_internal.h new file mode 100644 index 00000000..fb87605f --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/rec_backend_kernels_internal.h @@ -0,0 +1,30 @@ +#ifndef LW_REC_BACKEND_KERNELS_INTERNAL_H +#define LW_REC_BACKEND_KERNELS_INTERNAL_H + +#include "../kernels/nhwc_internal.h" + +typedef void (*lw_rec_ctc_argmax_scores_fn)( + const float*, const float*, const float*, uint32_t*, float*, + uint32_t, uint32_t, uint32_t); +typedef void (*lw_rec_ctc_probabilities_fn)( + const float*, const float*, const float*, const uint32_t*, const float*, + float*, uint32_t, uint32_t, uint32_t); + +typedef struct lw_rec_backend_kernels { + const char* name; + uint32_t vector_width; + uint32_t oc_block; + uint32_t has_fma; + void (*pointwise)(const float*, const float*, const lw_nhwc_epilogue*, + float*, uint32_t, uint32_t, uint32_t); + lw_status (*dense)(const float*, const float*, const lw_nhwc_epilogue*, + float*, const lw_nhwc_dense_desc*, void*, uint64_t); + lw_status (*depthwise)(const float*, const float*, const float*, + float*, const lw_nhwc_depthwise_desc*); + lw_rec_ctc_argmax_scores_fn ctc_argmax_scores; + lw_rec_ctc_probabilities_fn ctc_probabilities; +} lw_rec_backend_kernels; + +const lw_rec_backend_kernels* lw_rec_backend_kernels_current(void); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/session.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/session.c new file mode 100644 index 00000000..b43ff845 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/session.c @@ -0,0 +1,877 @@ +#include "session_internal.h" +#include "abi_compat_internal.h" + +/* + * Session construction joins model metadata, concrete input shapes and the + * workspace plan. The model bytes remain owned by lw_model; applications must + * therefore keep the model alive until every session created from it is freed. + */ +#include "lwm_read.h" +#include "packed_conv_internal.h" +#include "packed_conv3x3_internal.h" +#include "packed_matmul_internal.h" +#include "parallel_internal.h" +#include "cpu_features.h" +#include "operator_internal.h" + +#include +#include + +#if defined(_WIN32) +# include +#endif + +#define LW_DEFAULT_MAX_WORKSPACE_SIZE (UINT64_C(512) * UINT64_C(1024) * UINT64_C(1024)) +#define LW_DEFAULT_MAX_TENSOR_SIZE (UINT64_C(256) * UINT64_C(1024) * UINT64_C(1024)) + +static void* workspace_allocate(size_t size) { +#if defined(_WIN32) + return _aligned_malloc(size, LW_WORKSPACE_ALIGNMENT); +#else + void* pointer = NULL; + if (posix_memalign(&pointer, LW_WORKSPACE_ALIGNMENT, size) != 0) { + return NULL; + } + return pointer; +#endif +} + +static void workspace_release(void* pointer) { +#if defined(_WIN32) + _aligned_free(pointer); +#else + free(pointer); +#endif +} + +static void release_shared_prepared_constants(lw_shared_prepared_constants* constants) { + uint32_t observed; + + if (constants == NULL) { + return; + } + observed = lw_atomic_u32_load_acquire(&constants->ref_count); + for (;;) { + uint32_t desired; + if (observed == 0u) { + return; + } + desired = observed - 1u; + if (lw_atomic_u32_compare_exchange_acq_rel(&constants->ref_count, &observed, + desired)) { + if (desired != 0u) { + return; + } + break; + } + } + workspace_release(constants->packed_weights); + free(constants->constants); + free(constants); +} + +static int align_up_64(uint64_t value, uint64_t* aligned_value) { + const uint64_t mask = LW_WORKSPACE_ALIGNMENT - 1u; + if (aligned_value == NULL || value > UINT64_MAX - mask) { + return 0; + } + *aligned_value = (value + mask) & ~mask; + return 1; +} + +static int uint64_fits_size_t(uint64_t value) { + return (uint64_t)(size_t)value == value; +} + +static int prepared_pointwise_geometry(lw_simd_level simd_level, + const lw_runtime_tensor* input) { + const uint64_t height = (uint32_t)input->dimensions[2]; + const uint64_t width = (uint32_t)input->dimensions[3]; + if (width >= UINT64_C(2) * height) { + return lw_simd_level_has_packed_conv1x1(simd_level); + } +#if defined(_M_X64) || defined(__x86_64__) + /* The 4x16 microkernel needs the x64 register file. Keep square feature + * maps on the canonical kernel for x86, SSE2-only hosts and tiny tensors. */ + return lw_simd_level_is_avx2(simd_level) && height * width >= 256u; +#else + /* NEON and LSX both have enough vector registers for the portable packed + * four-output microkernel on large square detector feature maps. */ + return (lw_simd_level_is_neon(simd_level) || lw_simd_level_is_lsx(simd_level)) && + height * width >= 256u; +#endif +} + +static int prepared_pointwise_node(const lw_session* session, lw_simd_level simd_level, + uint32_t node_index, + uint32_t* weight_tensor_index, uint64_t* packed_weight_count) { + const lw_model* model = session->model; + const uint8_t* node = + model->bytes + (size_t)model->node_offset + (size_t)node_index * LWM_V0_NODE_SIZE; + uint16_t input_count = lwm_read_u16(node + 2u); + uint64_t param_offset; + const uint8_t* params; + uint32_t input_index; + uint32_t weights_index; + uint32_t output_index; + const lw_runtime_tensor* input; + const lw_runtime_tensor* weights; + const lw_runtime_tensor* output; + if (lwm_read_u16(node) != LW_OP_CONV || (input_count != 2u && input_count != 3u)) { + return 0; + } + param_offset = lwm_read_u64(node + 56u); + params = model->bytes + (size_t)param_offset; + input_index = lwm_read_u32(node + 8u); + weights_index = lwm_read_u32(node + 12u); + output_index = lwm_read_u32(node + 40u); + input = &session->tensors[input_index]; + weights = &session->tensors[weights_index]; + output = &session->tensors[output_index]; + /* Long OCR feature maps use the portable packed path. Large square maps + * additionally require the x64 AVX2 4x16 microkernel. */ + if (input->dtype != LW_DTYPE_F32 || input->rank != 4u || weights->dtype != LW_DTYPE_F32 || + weights->rank != 4u || output->dtype != LW_DTYPE_F32 || output->rank != 4u || + (weights->flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u || lwm_read_u32(params + 4u) != 1u || + lwm_read_i32(params + 8u) != 1 || lwm_read_i32(params + 12u) != 1 || + lwm_read_i32(params + 16u) != 1 || lwm_read_i32(params + 20u) != 1 || + lwm_read_i32(params + 24u) != 1 || lwm_read_i32(params + 28u) != 1 || + lwm_read_i32(params + 32u) != 0 || lwm_read_i32(params + 36u) != 0 || + lwm_read_i32(params + 40u) != 0 || lwm_read_i32(params + 44u) != 0 || + weights->dimensions[0] != output->dimensions[1] || + weights->dimensions[1] != input->dimensions[1] || weights->dimensions[2] != 1 || + weights->dimensions[3] != 1 || input->dimensions[0] != output->dimensions[0] || + output->dimensions[1] < (int32_t)LW_PACKED_CONV1X1_OUTPUT_TILE || + output->dimensions[1] % (int32_t)LW_PACKED_CONV1X1_OUTPUT_TILE != 0 || + !prepared_pointwise_geometry(simd_level, input) || + input->dimensions[2] != output->dimensions[2] || + input->dimensions[3] != output->dimensions[3] || + !lw_packed_conv1x1_weight_count((uint32_t)input->dimensions[1], + (uint32_t)output->dimensions[1], packed_weight_count)) { + return 0; + } + *weight_tensor_index = weights_index; + return 1; +} + +static int prepared_matmul_node(const lw_session* session, lw_simd_level simd_level, + uint32_t node_index, uint32_t* weight_tensor_index, + uint64_t* packed_weight_count) { +#if defined(_M_X64) || defined(__x86_64__) + const lw_model* model = session->model; + const uint8_t* node = + model->bytes + (size_t)model->node_offset + (size_t)node_index * LWM_V0_NODE_SIZE; + uint32_t input_index; + uint32_t weights_index; + uint32_t output_index; + const lw_runtime_tensor* input; + const lw_runtime_tensor* weights; + const lw_runtime_tensor* output; + uint32_t rows; + uint32_t inner_dimension; + uint32_t columns; + if (!lw_simd_level_is_avx2(simd_level) || lwm_read_u16(node) != LW_OP_MATMUL || + lwm_read_u16(node + 2u) != 2u) { + return 0; + } + input_index = lwm_read_u32(node + 8u); + weights_index = lwm_read_u32(node + 12u); + output_index = lwm_read_u32(node + 40u); + input = &session->tensors[input_index]; + weights = &session->tensors[weights_index]; + output = &session->tensors[output_index]; + if (input->dtype != LW_DTYPE_F32 || input->rank < 2u || + weights->dtype != LW_DTYPE_F32 || weights->rank != 2u || + (weights->flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u || output->dtype != LW_DTYPE_F32 || + output->rank != input->rank) { + return 0; + } + rows = (uint32_t)input->dimensions[input->rank - 2u]; + inner_dimension = (uint32_t)input->dimensions[input->rank - 1u]; + columns = (uint32_t)weights->dimensions[1]; + if (rows < 4u || rows % 4u != 0u || inner_dimension < 64u || columns < 1024u || + weights->dimensions[0] != (int32_t)inner_dimension || + output->dimensions[output->rank - 2u] != (int32_t)rows || + output->dimensions[output->rank - 1u] != (int32_t)columns || + !lw_packed_matmul_weight_count(inner_dimension, columns, packed_weight_count)) { + return 0; + } + *weight_tensor_index = weights_index; + return 1; +#else + (void)session; + (void)simd_level; + (void)node_index; + (void)weight_tensor_index; + (void)packed_weight_count; + return 0; +#endif +} + +static int prepared_stride2_conv3x3_node( + const lw_session* session, lw_simd_level simd_level, uint32_t node_index, + uint32_t* weight_tensor_index, uint64_t* packed_weight_count) { +#if defined(_M_X64) || defined(__x86_64__) + const lw_model* model = session->model; + const uint8_t* node = + model->bytes + (size_t)model->node_offset + (size_t)node_index * LWM_V0_NODE_SIZE; + const uint16_t input_count = lwm_read_u16(node + 2u); + uint64_t param_offset; + const uint8_t* params; + uint32_t input_index; + uint32_t weights_index; + uint32_t output_index; + const lw_runtime_tensor* input; + const lw_runtime_tensor* weights; + const lw_runtime_tensor* output; + if (!lw_simd_level_is_avx2(simd_level) || lwm_read_u16(node) != LW_OP_CONV || + (input_count != 2u && input_count != 3u)) { + return 0; + } + param_offset = lwm_read_u64(node + 56u); + params = model->bytes + (size_t)param_offset; + input_index = lwm_read_u32(node + 8u); + weights_index = lwm_read_u32(node + 12u); + output_index = lwm_read_u32(node + 40u); + input = &session->tensors[input_index]; + weights = &session->tensors[weights_index]; + output = &session->tensors[output_index]; + if (input->dtype != LW_DTYPE_F32 || input->rank != 4u || + weights->dtype != LW_DTYPE_F32 || weights->rank != 4u || + output->dtype != LW_DTYPE_F32 || output->rank != 4u || + (weights->flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u || + lwm_read_u32(params + 4u) != 1u || lwm_read_i32(params + 8u) != 3 || + lwm_read_i32(params + 12u) != 3 || lwm_read_i32(params + 16u) != 2 || + lwm_read_i32(params + 20u) != 2 || lwm_read_i32(params + 24u) != 1 || + lwm_read_i32(params + 28u) != 1 || lwm_read_i32(params + 32u) != 1 || + lwm_read_i32(params + 36u) != 1 || lwm_read_i32(params + 40u) != 1 || + lwm_read_i32(params + 44u) != 1 || + weights->dimensions[0] != output->dimensions[1] || + weights->dimensions[1] != input->dimensions[1] || weights->dimensions[2] != 3 || + weights->dimensions[3] != 3 || input->dimensions[0] != output->dimensions[0] || + (uint64_t)(uint32_t)input->dimensions[3] < + UINT64_C(2) * (uint32_t)input->dimensions[2] || + output->dimensions[1] < (int32_t)LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE || + output->dimensions[1] % + (int32_t)LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE != + 0 || + !lw_packed_conv3x3_stride2_weight_count( + (uint32_t)input->dimensions[1], (uint32_t)output->dimensions[1], + packed_weight_count)) { + return 0; + } + *weight_tensor_index = weights_index; + return 1; +#else + (void)session; + (void)simd_level; + (void)node_index; + (void)weight_tensor_index; + (void)packed_weight_count; + return 0; +#endif +} + +static const float* constant_f32_data(const lw_session* session, uint32_t tensor_index) { + const lw_model* model = session->model; + const uint8_t* disk = + model->bytes + (size_t)model->tensor_offset + (size_t)tensor_index * LWM_V0_TENSOR_SIZE; + uint64_t data_offset = lwm_read_u64(disk + 48u); + return (const float*)(const void*)(model->bytes + (size_t)data_offset); +} + +static lw_status prepare_constant_layout(lw_session* session, lw_error* error) { + const lw_model* model = session->model; + const lw_simd_level simd_level = session->cpu.simd; + uint64_t total_bytes = 0u; + uint32_t node_index; + if (model->info.node_count == 0u || + !lw_simd_level_has_packed_conv1x1(simd_level)) { + return LW_STATUS_OK; + } + session->shared_prepared_constants = + (lw_shared_prepared_constants*)calloc(1u, sizeof(*session->shared_prepared_constants)); + if (session->shared_prepared_constants == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, + "unable to allocate shared prepared constants"); + return LW_STATUS_OUT_OF_MEMORY; + } + lw_atomic_u32_init(&session->shared_prepared_constants->ref_count, 1u); + session->prepared_constants = + (lw_prepared_constant*)calloc(model->info.node_count, sizeof(*session->prepared_constants)); + if (session->prepared_constants == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate prepared node table"); + return LW_STATUS_OUT_OF_MEMORY; + } + session->shared_prepared_constants->constants = session->prepared_constants; + for (node_index = 0u; node_index < model->info.node_count; ++node_index) { + uint32_t weight_tensor_index; + uint64_t packed_weight_count; + uint64_t packed_bytes; + lw_prepared_constant* prepared = &session->prepared_constants[node_index]; + if (prepared_pointwise_node(session, simd_level, node_index, &weight_tensor_index, + &packed_weight_count)) { + prepared->kind = LW_PREPARED_CONSTANT_CONV1X1_PACKED4; + } else if (prepared_stride2_conv3x3_node( + session, simd_level, node_index, &weight_tensor_index, + &packed_weight_count)) { + prepared->kind = LW_PREPARED_CONSTANT_CONV3X3_STRIDE2_PACKED8; + } else if (prepared_matmul_node(session, simd_level, node_index, &weight_tensor_index, + &packed_weight_count)) { + prepared->kind = LW_PREPARED_CONSTANT_MATMUL_PACKED16; + } else { + continue; + } + (void)weight_tensor_index; + packed_bytes = packed_weight_count * sizeof(float); + if (!align_up_64(total_bytes, &total_bytes)) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "packed weight alignment overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + if (total_bytes > UINT64_MAX - packed_bytes || + !uint64_fits_size_t(total_bytes + packed_bytes)) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "packed weights overflow"); + return LW_STATUS_OUT_OF_BOUNDS; + } + prepared->packed_weight_offset = total_bytes; + prepared->packed_weight_count = packed_weight_count; + total_bytes += packed_bytes; + } + if (total_bytes == 0u) { + return LW_STATUS_OK; + } + if (!align_up_64(total_bytes, &total_bytes) || !uint64_fits_size_t(total_bytes)) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "packed weight allocation overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + session->packed_weight_bytes = (size_t)total_bytes; + session->shared_prepared_constants->packed_weight_bytes = session->packed_weight_bytes; + return LW_STATUS_OK; +} + +static lw_status pack_prepared_constants(lw_session* session, lw_error* error) { + const lw_model* model = session->model; + const lw_simd_level simd_level = session->cpu.simd; + uint32_t node_index; + if (session->packed_weight_bytes == 0u) { + return LW_STATUS_OK; + } + session->packed_weights = (uint8_t*)workspace_allocate(session->packed_weight_bytes); + if (session->packed_weights == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate packed weights"); + return LW_STATUS_OUT_OF_MEMORY; + } + session->shared_prepared_constants->packed_weights = session->packed_weights; + for (node_index = 0u; node_index < model->info.node_count; ++node_index) { + lw_prepared_constant* prepared = &session->prepared_constants[node_index]; + uint32_t weight_tensor_index; + uint64_t packed_weight_count; + const uint8_t* node; + uint32_t input_index; + uint32_t output_index; + node = model->bytes + (size_t)model->node_offset + (size_t)node_index * LWM_V0_NODE_SIZE; + if (prepared->kind == LW_PREPARED_CONSTANT_CONV1X1_PACKED4 && + prepared_pointwise_node(session, simd_level, node_index, &weight_tensor_index, + &packed_weight_count) && + packed_weight_count == prepared->packed_weight_count) { + input_index = lwm_read_u32(node + 8u); + output_index = lwm_read_u32(node + 40u); + lw_pack_conv1x1_weights_f32( + constant_f32_data(session, weight_tensor_index), + (uint32_t)session->tensors[input_index].dimensions[1], + (uint32_t)session->tensors[output_index].dimensions[1], + (float*)(void*)(session->packed_weights + + (size_t)prepared->packed_weight_offset)); + } else if (prepared->kind == LW_PREPARED_CONSTANT_CONV3X3_STRIDE2_PACKED8 && + prepared_stride2_conv3x3_node( + session, simd_level, node_index, &weight_tensor_index, + &packed_weight_count) && + packed_weight_count == prepared->packed_weight_count) { + input_index = lwm_read_u32(node + 8u); + output_index = lwm_read_u32(node + 40u); + lw_pack_conv3x3_stride2_weights_f32( + constant_f32_data(session, weight_tensor_index), + (uint32_t)session->tensors[input_index].dimensions[1], + (uint32_t)session->tensors[output_index].dimensions[1], + (float*)(void*)(session->packed_weights + + (size_t)prepared->packed_weight_offset)); + } else if (prepared->kind == LW_PREPARED_CONSTANT_MATMUL_PACKED16 && + prepared_matmul_node(session, simd_level, node_index, &weight_tensor_index, + &packed_weight_count) && + packed_weight_count == prepared->packed_weight_count) { + const lw_runtime_tensor* weights = &session->tensors[weight_tensor_index]; + lw_pack_matmul_weights_f32( + constant_f32_data(session, weight_tensor_index), + (uint32_t)weights->dimensions[0], (uint32_t)weights->dimensions[1], + (float*)(void*)(session->packed_weights + + (size_t)prepared->packed_weight_offset)); + } + } + return LW_STATUS_OK; +} + +static lw_status prepare_constant_weights(lw_session* session, lw_error* error) { + lw_status status = prepare_constant_layout(session, error); + if (status != LW_STATUS_OK) { + return status; + } + return pack_prepared_constants(session, error); +} + +static lw_status prepare_or_share_constant_weights(lw_session* session, + const lw_session* source, + lw_error* error) { + lw_status status; + if (source == NULL) { + return prepare_constant_weights(session, error); + } + status = prepare_constant_layout(session, error); + if (status != LW_STATUS_OK) { + return status; + } + if (lw_session_prepared_constants_compatible(session, source)) { + /* The candidate has only a descriptor table at this point. Replace it + * with the source's immutable arena before any packed allocation. */ + return lw_session_share_prepared_constants(session, source, error); + } + return pack_prepared_constants(session, error); +} + +static uint32_t runtime_dtype_size(uint32_t dtype) { + switch (dtype) { + case LW_DTYPE_F32: + return 4u; + case LW_DTYPE_I32: + return 4u; + case LW_DTYPE_I64: + return 8u; + case LW_DTYPE_U8: + return 1u; + default: + return 0u; + } +} + +static lw_status resolve_existing_tensor_size(lw_runtime_tensor* tensor, uint64_t max_tensor_size, + lw_error* error) { + uint64_t elements = 1u; + uint32_t item_size = runtime_dtype_size(tensor->dtype); + uint32_t i; + if (item_size == 0u || tensor->rank > LW_MAX_DIMS) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "input or constant tensor type is invalid"); + return LW_STATUS_INVALID_SHAPE; + } + for (i = 0u; i < tensor->rank; ++i) { + int32_t dimension = tensor->dimensions[i]; + if (dimension <= 0 || elements > UINT64_MAX / (uint32_t)dimension) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "input or constant tensor dimensions are invalid"); + return LW_STATUS_INVALID_SHAPE; + } + elements *= (uint32_t)dimension; + } + if (elements > UINT64_MAX / item_size) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "input or constant tensor byte size overflows"); + return LW_STATUS_INVALID_SHAPE; + } + tensor->byte_size = elements * item_size; + if (tensor->byte_size > max_tensor_size) { + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, + "input or constant tensor exceeds max_tensor_size"); + return LW_STATUS_MEMORY_LIMIT; + } + return LW_STATUS_OK; +} + +void lw_tensor_desc_init(lw_tensor_desc* tensor) { + if (tensor == NULL) { + return; + } + memset(tensor, 0, sizeof(*tensor)); + tensor->struct_size = (uint32_t)sizeof(*tensor); +} + +void lw_session_options_init(lw_session_options* options) { + if (options == NULL) { + return; + } + memset(options, 0, sizeof(*options)); + options->struct_size = (uint32_t)sizeof(*options); + options->max_workspace_size = LW_DEFAULT_MAX_WORKSPACE_SIZE; + options->max_tensor_size = LW_DEFAULT_MAX_TENSOR_SIZE; +} + +void lw_session_info_init(lw_session_info* info) { + if (info == NULL) { + return; + } + memset(info, 0, sizeof(*info)); + info->struct_size = (uint32_t)sizeof(*info); +} + +static lw_status lw_session_create_with_plan_flags(const lw_model* model, + const lw_tensor_desc* inputs, + uint32_t input_count, + const lw_session_options* options, + uint32_t plan_flags, + const lw_session* prepared_source, + lw_session** out_session, lw_error* error) { + uint64_t max_workspace_size = LW_DEFAULT_MAX_WORKSPACE_SIZE; + uint64_t max_tensor_size = LW_DEFAULT_MAX_TENSOR_SIZE; + lw_session* session; + uint32_t i; + lw_status status; + if (out_session != NULL) { + *out_session = NULL; + } + if (model == NULL || inputs == NULL || out_session == NULL || + input_count != model->info.input_count) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "model, inputs, matching input_count, and out_session are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (options != NULL) { + if (options->struct_size != sizeof(*options) || options->reserved != 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid session options structure"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (options->max_workspace_size != 0u) { + max_workspace_size = options->max_workspace_size; + } + if (options->max_tensor_size != 0u) { + max_tensor_size = options->max_tensor_size; + } + } + session = (lw_session*)calloc(1u, sizeof(*session)); + if (session == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate session handle"); + return LW_STATUS_OUT_OF_MEMORY; + } + session->intra_op_thread_count = 1u; + session->model = model; + session->cpu = lw_get_cpu_capabilities(); + session->ctc_greedy_plan_enabled = + (plan_flags & LW_SESSION_PLAN_CTC_GREEDY) != 0u ? 1u : 0u; + session->ctc_greedy_skip_tensor = UINT32_MAX; + session->tensors = + (lw_runtime_tensor*)calloc(model->info.tensor_count, sizeof(*session->tensors)); + if (session->tensors == NULL) { + lw_session_free(session); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate resolved tensor table"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (i = 0u; i < model->info.tensor_count; ++i) { + const uint8_t* disk = + model->bytes + (size_t)model->tensor_offset + (size_t)i * LWM_V0_TENSOR_SIZE; + lw_runtime_tensor* tensor = &session->tensors[i]; + uint32_t j; + tensor->dtype = lwm_read_u32(disk); + tensor->rank = lwm_read_u32(disk + 4); + tensor->flags = lwm_read_u32(disk + 40); + tensor->workspace_offset = UINT64_MAX; + for (j = 0u; j < LW_MAX_DIMS; ++j) { + tensor->dimensions[j] = lwm_read_i32(disk + 8u + j * 4u); + } + } + /* Replace model -1 dimensions with the caller's concrete input shape while + * requiring every static dimension to match exactly. */ + for (i = 0u; i < input_count; ++i) { + uint32_t tensor_index = + lwm_read_u32(model->bytes + (size_t)model->input_offset + (size_t)i * 4u); + lw_runtime_tensor* tensor = &session->tensors[tensor_index]; + const lw_tensor_desc* input = &inputs[i]; + uint32_t j; + if (input->struct_size != sizeof(*input) || input->reserved != 0u || + input->dtype != tensor->dtype || input->rank != tensor->rank) { + lw_session_free(session); + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "input tensor descriptor type or rank is invalid"); + return LW_STATUS_INVALID_SHAPE; + } + for (j = 0u; j < input->rank; ++j) { + if (input->dimensions[j] <= 0 || + (tensor->dimensions[j] != -1 && tensor->dimensions[j] != input->dimensions[j])) { + lw_session_free(session); + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "input tensor dimensions disagree with the model"); + return LW_STATUS_INVALID_SHAPE; + } + } + for (j = input->rank; j < LW_MAX_DIMS; ++j) { + if (input->dimensions[j] != 0) { + lw_session_free(session); + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "unused input tensor dimensions must be zero"); + return LW_STATUS_INVALID_SHAPE; + } + } + memcpy(tensor->dimensions, input->dimensions, sizeof(tensor->dimensions)); + } + for (i = 0u; i < model->info.tensor_count; ++i) { + lw_runtime_tensor* tensor = &session->tensors[i]; + if ((tensor->flags & (LWM_V0_TENSOR_FLAG_CONSTANT | LWM_V0_TENSOR_FLAG_INPUT)) != 0u) { + status = resolve_existing_tensor_size(tensor, max_tensor_size, error); + if (status != LW_STATUS_OK) { + lw_session_free(session); + return status; + } + } + } + /* Shape resolution precedes planning because every workspace allocation + * needs a concrete, overflow-checked byte size. */ + status = lw_resolve_shapes(session, max_tensor_size, error); + if (status != LW_STATUS_OK) { + lw_session_free(session); + return status; + } + status = lw_prepare_execution_plan(session, error); + if (status != LW_STATUS_OK) { + lw_session_free(session); + return status; + } + status = lw_plan_workspace(session, max_workspace_size, error); + if (status != LW_STATUS_OK) { + lw_session_free(session); + return status; + } + if (session->workspace_bytes != 0u && + (plan_flags & LW_SESSION_PLAN_METADATA_ONLY) == 0u) { + session->workspace = (uint8_t*)workspace_allocate(session->workspace_bytes); + if (session->workspace == NULL) { + lw_session_free(session); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, + "unable to allocate planned session workspace"); + return LW_STATUS_OUT_OF_MEMORY; + } + } + if ((plan_flags & LW_SESSION_PLAN_METADATA_ONLY) == 0u) { + status = prepare_or_share_constant_weights(session, prepared_source, error); + if (status != LW_STATUS_OK) { + lw_session_free(session); + return status; + } + status = lw_prepare_execution_nodes(session, error); + if (status != LW_STATUS_OK) { + lw_session_free(session); + return status; + } + } + memset(&session->info, 0, sizeof(session->info)); + session->info.struct_size = (uint32_t)sizeof(session->info); + session->info.tensor_count = model->info.tensor_count; + session->info.input_count = model->info.input_count; + session->info.output_count = model->info.output_count; + session->info.workspace_size = session->workspace_bytes; + *out_session = session; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +lw_status lw_session_create(const lw_model* model, const lw_tensor_desc* inputs, + uint32_t input_count, const lw_session_options* options, + lw_session** out_session, lw_error* error) { + return lw_session_create_with_plan_flags(model, inputs, input_count, options, 0u, NULL, + out_session, error); +} + +lw_status lw_session_create_ctc_greedy(const lw_model* model, const lw_tensor_desc* inputs, + uint32_t input_count, const lw_session_options* options, + lw_session** out_session, lw_error* error) { + return lw_session_create_with_plan_flags(model, inputs, input_count, options, + LW_SESSION_PLAN_CTC_GREEDY, NULL, out_session, + error); +} + +lw_status lw_session_create_metadata_only(const lw_model* model, + const lw_tensor_desc* inputs, uint32_t input_count, + const lw_session_options* options, + lw_session** out_session, lw_error* error) { + return lw_session_create_with_plan_flags(model, inputs, input_count, options, + LW_SESSION_PLAN_METADATA_ONLY, NULL, + out_session, error); +} + +lw_status lw_session_create_with_prepared_source( + const lw_model* model, const lw_tensor_desc* inputs, uint32_t input_count, + const lw_session_options* options, uint32_t plan_flags, const lw_session* prepared_source, + lw_session** out_session, lw_error* error) { + return lw_session_create_with_plan_flags(model, inputs, input_count, options, plan_flags, + prepared_source, out_session, error); +} + +void lw_session_set_intra_op_thread_count(lw_session* session, uint32_t thread_count) { + if (session == NULL) { + return; + } + if (thread_count == 0u) { + thread_count = 1u; + } + if (thread_count > LW_PARALLEL_MAX_WORKERS) { + thread_count = LW_PARALLEL_MAX_WORKERS; + } + if (session->intra_op_thread_count == thread_count) { + return; + } + lw_thread_pool_free(session->thread_pool); + session->thread_pool = thread_count > 1u ? lw_thread_pool_create(thread_count) : NULL; + session->intra_op_thread_count = lw_thread_pool_worker_count(session->thread_pool); +} + +void lw_session_set_external_thread_pool(lw_session* session, lw_thread_pool* pool, + uint32_t worker_count) { + if (session == NULL) { + return; + } + if (pool == NULL || worker_count <= 1u) { + pool = NULL; + worker_count = 0u; + } + session->external_thread_pool = pool; + session->external_worker_count = worker_count; +} + +lw_thread_pool* lw_session_ctc_head_pool(const lw_session* session, uint32_t* out_workers) { + if (out_workers != NULL) { + *out_workers = 1u; + } + if (session == NULL) { + return NULL; + } + if (session->external_thread_pool != NULL && session->external_worker_count > 1u) { + if (out_workers != NULL) { + *out_workers = session->external_worker_count; + } + return session->external_thread_pool; + } + if (session->thread_pool != NULL && session->intra_op_thread_count > 1u) { + if (out_workers != NULL) { + *out_workers = session->intra_op_thread_count; + } + return session->thread_pool; + } + return NULL; +} + +void lw_session_free(lw_session* session) { + if (session == NULL) { + return; + } + lw_thread_pool_free(session->thread_pool); + session->thread_pool = NULL; + lw_free_execution_nodes(session); + lw_free_execution_plan(session); + workspace_release(session->workspace); + session->workspace = NULL; + release_shared_prepared_constants(session->shared_prepared_constants); + session->shared_prepared_constants = NULL; + session->packed_weights = NULL; + session->packed_weight_bytes = 0u; + session->prepared_constants = NULL; + free(session->tensors); + session->tensors = NULL; + free(session); +} + +lw_status lw_session_share_prepared_constants(lw_session* destination, + const lw_session* source, + lw_error* error) { + lw_status execution_status; + lw_shared_prepared_constants* shared; + uint32_t observed; + + if (destination == NULL || source == NULL || destination->model != source->model) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, + "sessions from the same model are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (source->shared_prepared_constants == NULL) { + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + } + if (destination == source || + destination->shared_prepared_constants == source->shared_prepared_constants) { + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; + } + shared = source->shared_prepared_constants; + observed = lw_atomic_u32_load_acquire(&shared->ref_count); + for (;;) { + if (observed == 0u || observed == UINT32_MAX) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, + "prepared constants reference count overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + if (lw_atomic_u32_compare_exchange_acq_rel(&shared->ref_count, &observed, + observed + 1u)) { + break; + } + } + lw_free_execution_nodes(destination); + release_shared_prepared_constants(destination->shared_prepared_constants); + destination->shared_prepared_constants = shared; + destination->prepared_constants = shared->constants; + destination->packed_weights = shared->packed_weights; + destination->packed_weight_bytes = shared->packed_weight_bytes; + execution_status = lw_prepare_execution_nodes(destination, error); + if (execution_status != LW_STATUS_OK) { + return execution_status; + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} +int lw_session_prepared_constants_compatible(const lw_session* destination, + const lw_session* source) { + uint32_t node_index; + if (destination == NULL || source == NULL || destination == source || + destination->model != source->model || destination->cpu.simd != source->cpu.simd || + destination->model->info.node_count != source->model->info.node_count || + destination->shared_prepared_constants == NULL || + source->shared_prepared_constants == NULL || + destination->packed_weight_bytes != source->packed_weight_bytes) { + return 0; + } + if (destination->prepared_constants == NULL || source->prepared_constants == NULL) { + return 0; + } + for (node_index = 0u; node_index < destination->model->info.node_count; ++node_index) { + const lw_prepared_constant* left = &destination->prepared_constants[node_index]; + const lw_prepared_constant* right = &source->prepared_constants[node_index]; + if (left->kind != right->kind || left->packed_weight_offset != right->packed_weight_offset || + left->packed_weight_count != right->packed_weight_count) { + return 0; + } + } + return 1; +} + +lw_status lw_session_get_info(const lw_session* session, lw_session_info* info) { + if (session == NULL || info == NULL || + !lw_abi_copy_output_prefix(info, info->struct_size, &session->info, sizeof(session->info))) { + return LW_STATUS_INVALID_ARGUMENT; + } + return LW_STATUS_OK; +} + +lw_status lw_session_get_output_desc(const lw_session* session, uint32_t output_index, + lw_tensor_desc* output) { + uint32_t tensor_index; + const lw_runtime_tensor* tensor; + lw_tensor_desc value; + if (session == NULL || output == NULL || output->struct_size < sizeof(uint32_t) || + output_index >= session->model->info.output_count) { + return LW_STATUS_INVALID_ARGUMENT; + } + tensor_index = lwm_read_u32(session->model->bytes + (size_t)session->model->output_offset + + (size_t)output_index * 4u); + tensor = &session->tensors[tensor_index]; + memset(&value, 0, sizeof(value)); + value.struct_size = (uint32_t)sizeof(value); + value.dtype = tensor->dtype; + value.rank = tensor->rank; + memcpy(value.dimensions, tensor->dimensions, sizeof(value.dimensions)); + if (!lw_abi_copy_output_prefix(output, output->struct_size, &value, sizeof(value))) { + return LW_STATUS_INVALID_ARGUMENT; + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/session_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/session_internal.h new file mode 100644 index 00000000..ceb9408b --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/session_internal.h @@ -0,0 +1,190 @@ +#ifndef LW_SESSION_INTERNAL_H +#define LW_SESSION_INTERNAL_H + +/* Runtime tensor metadata shared by shape inference, planning and execution. */ + +#include "model_internal.h" +#include "parallel_internal.h" +#include "atomic_internal.h" +#include "../simd/cpu_features.h" +#include "../kernels/packed_conv_internal.h" +#include "../kernels/packed_conv3x3_internal.h" + +#include +#include + +#define LW_WORKSPACE_ALIGNMENT 64u +#define LW_SESSION_PLAN_CTC_GREEDY UINT32_C(1) +#define LW_SESSION_PLAN_METADATA_ONLY UINT32_C(2) + +typedef enum lw_prepared_constant_kind { + LW_PREPARED_CONSTANT_NONE = 0, + LW_PREPARED_CONSTANT_CONV1X1_PACKED4 = 1, + LW_PREPARED_CONSTANT_MATMUL_PACKED16 = 2, + LW_PREPARED_CONSTANT_CONV3X3_STRIDE2_PACKED8 = 3 +} lw_prepared_constant_kind; + +typedef struct lw_prepared_constant { + uint32_t kind; + uint32_t reserved; + uint64_t packed_weight_offset; + uint64_t packed_weight_count; +} lw_prepared_constant; + +typedef struct lw_shared_prepared_constants { + lw_atomic_u32 ref_count; + lw_prepared_constant* constants; + uint8_t* packed_weights; + size_t packed_weight_bytes; +} lw_shared_prepared_constants; +typedef enum lw_bound_execution_kind { + LW_BOUND_EXEC_GENERIC = 0, + LW_BOUND_EXEC_CONV1X1_PACKED = 1, + LW_BOUND_EXEC_CONV3X3_PACKED = 2 +} lw_bound_execution_kind; + +typedef enum lw_conv1x1_kernel_id { + LW_CONV1X1_KERNEL_NONE = 0, + LW_CONV1X1_KERNEL_SCALAR = 1, + LW_CONV1X1_KERNEL_SSE2 = 2, + LW_CONV1X1_KERNEL_AVX2 = 3, + LW_CONV1X1_KERNEL_NEON = 4, + LW_CONV1X1_KERNEL_LSX = 5 +} lw_conv1x1_kernel_id; + +typedef enum lw_conv3x3_kernel_id { + LW_CONV3X3_KERNEL_NONE = 0, + LW_CONV3X3_KERNEL_SCALAR = 1, + LW_CONV3X3_KERNEL_AVX2 = 2 +} lw_conv3x3_kernel_id; + +typedef struct lw_bound_conv3x3 { + lw_packed_conv3x3_kernel_fn kernel; + const float* packed_weights; + uint32_t input_index; + uint32_t bias_index; + uint32_t output_index; + uint16_t kernel_id; + uint16_t output_tile; +} lw_bound_conv3x3; + +typedef struct lw_bound_conv1x1 { + lw_packed_conv1x1_kernel_fn kernel; + const float* packed_weights; + uint32_t input_index; + uint32_t bias_index; + uint32_t output_index; + uint16_t kernel_id; + uint16_t output_tile; +} lw_bound_conv1x1; + +typedef struct lw_bound_node { + uint32_t node_index; + uint16_t operator_type; + uint16_t input_count; + uint32_t input_indices[LWM_V0_MAX_NODE_INPUTS]; + uint32_t output_index; + uint32_t implementation; + uint16_t execution_kind; + uint16_t reserved_execution; + union { + lw_bound_conv1x1 conv1x1; + lw_bound_conv3x3 conv3x3; + } data; + const lw_prepared_constant* prepared_constant; +} lw_bound_node; + +typedef struct lw_runtime_tensor { + uint32_t dtype; + uint32_t rank; + int32_t dimensions[LW_MAX_DIMS]; + uint32_t flags; + uint64_t byte_size; + uint64_t workspace_offset; + int32_t birth_node; + int32_t last_use_node; + int workspace_live; +} lw_runtime_tensor; + +/* Semantic match shared by the physical plan and the executor. Keeping the + * graph pattern in one helper prevents the planner from dropping a temporary + * that the runtime would still materialize (or vice versa). */ +typedef struct lw_fused_gelu_match { + uint32_t inputs[5][2]; + uint32_t outputs[5]; +} lw_fused_gelu_match; + +struct lw_session { + const lw_model* model; + lw_cpu_capabilities cpu; + lw_runtime_tensor* tensors; + uint8_t* workspace; + size_t workspace_bytes; + lw_prepared_constant* prepared_constants; + uint8_t* packed_weights; + size_t packed_weight_bytes; + lw_shared_prepared_constants* shared_prepared_constants; + lw_bound_node* execution_nodes; + uint32_t execution_node_count; + uint8_t* fused_gelu_start_nodes; + uint8_t* fused_gelu_skip_tensors; + uint8_t ctc_greedy_plan_enabled; + uint32_t ctc_greedy_skip_tensor; + float* ctc_logits_scratch; + uint32_t ctc_logits_scratch_rows; + uint64_t ctc_logits_scratch_elements; + lw_thread_pool* thread_pool; + uint32_t intra_op_thread_count; + /* Borrowed pool used only by the CTC head projection: lets the recognizer + * share one intra-op pool across its width-slot sessions instead of each + * session owning a full pool. Never owned by the session. */ + lw_thread_pool* external_thread_pool; + uint32_t external_worker_count; + lw_session_info info; +}; + +lw_status lw_resolve_shapes(lw_session* session, uint64_t max_tensor_size, lw_error* error); +/* Importer-only shape bridge: validated indices, no numeric execution. */ +lw_status lw_resolve_import_node(lw_session* session, const uint8_t* node, + uint64_t max_tensor_size, lw_error* error); +lw_status lw_plan_workspace(lw_session* session, uint64_t max_workspace_size, lw_error* error); +void lw_session_set_intra_op_thread_count(lw_session* session, uint32_t thread_count); +/* Borrow an external pool for the CTC head projection (worker_count <= 1 or + * NULL pool detaches). The pool must outlive the session and is never freed + * by the session. */ +void lw_session_set_external_thread_pool(lw_session* session, lw_thread_pool* pool, + uint32_t worker_count); +/* Pool the CTC head projection may fan out to: the borrowed pool when + * attached, else the session's own intra-op pool. *out_workers receives the + * usable worker count (1 when serial). */ +lw_thread_pool* lw_session_ctc_head_pool(const lw_session* session, uint32_t* out_workers); +lw_status lw_session_share_prepared_constants(lw_session* destination, + const lw_session* source, + lw_error* error); +int lw_session_prepared_constants_compatible(const lw_session* destination, + const lw_session* source); +lw_status lw_prepare_execution_nodes(lw_session* session, lw_error* error); +void lw_free_execution_nodes(lw_session* session); +lw_status lw_prepare_execution_plan(lw_session* session, lw_error* error); +void lw_free_execution_plan(lw_session* session); +lw_status lw_session_create_ctc_greedy(const lw_model* model, const lw_tensor_desc* inputs, + uint32_t input_count, const lw_session_options* options, + lw_session** out_session, lw_error* error); +/* Resolve shapes and the workspace plan without allocating execution buffers. + * A metadata-only session must never be passed to the canonical executor. */ +lw_status lw_session_create_metadata_only(const lw_model* model, + const lw_tensor_desc* inputs, uint32_t input_count, + const lw_session_options* options, + lw_session** out_session, lw_error* error); +lw_status lw_session_create_with_prepared_source( + const lw_model* model, const lw_tensor_desc* inputs, uint32_t input_count, + const lw_session_options* options, uint32_t plan_flags, const lw_session* prepared_source, + lw_session** out_session, lw_error* error); +int lw_match_fused_gelu(const lw_session* session, uint32_t node_index, + lw_fused_gelu_match* match); +int lw_execution_plan_is_fused_gelu_start(const lw_session* session, uint32_t node_index); +int lw_execution_plan_is_fused_gelu_skip(const lw_session* session, uint32_t tensor_index); +int lw_execution_plan_is_ctc_greedy_skip(const lw_session* session, uint32_t tensor_index); +int lw_execution_plan_is_skipped_tensor(const lw_session* session, uint32_t tensor_index); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/shape.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/shape.c new file mode 100644 index 00000000..ff2d6650 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/shape.c @@ -0,0 +1,585 @@ +#include "session_internal.h" + +/* + * Static and input-dependent shape inference for the supported LWM operators. + * Besides dimensions, this pass computes bounded tensor byte sizes consumed by + * the workspace planner. It performs no numeric inference work. + */ +#include "lwm_read.h" + +#include +#include +#include + +static float read_f32(const uint8_t* bytes) { + uint32_t bits = lwm_read_u32(bytes); + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static lw_status shape_fail(lw_error* error, const char* message) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, message); + return LW_STATUS_INVALID_SHAPE; +} + +static uint32_t dtype_size(uint32_t dtype) { + switch (dtype) { + case LW_DTYPE_F32: + return 4u; + case LW_DTYPE_I32: + return 4u; + case LW_DTYPE_I64: + return 8u; + case LW_DTYPE_U8: + return 1u; + default: + return 0u; + } +} + +static lw_status set_tensor_shape(lw_runtime_tensor* tensor, uint32_t dtype, uint32_t rank, + const int32_t* dimensions, uint64_t max_tensor_size, + lw_error* error) { + uint64_t elements = 1u; + uint32_t item_size = dtype_size(dtype); + uint32_t i; + if (item_size == 0u || rank > LW_MAX_DIMS) { + return shape_fail(error, "unsupported resolved tensor dtype or rank"); + } + if (tensor->dtype != dtype || tensor->rank != rank) { + return shape_fail(error, "resolved tensor type or rank disagrees with the model"); + } + for (i = 0u; i < rank; ++i) { + int32_t expected = tensor->dimensions[i]; + int32_t resolved = dimensions[i]; + if (resolved <= 0 || (expected != -1 && expected != resolved)) { + return shape_fail(error, "resolved tensor dimension disagrees with the model"); + } + if (elements > UINT64_MAX / (uint32_t)resolved) { + return shape_fail(error, "resolved tensor element count overflows"); + } + elements *= (uint32_t)resolved; + } + if (elements > UINT64_MAX / item_size) { + return shape_fail(error, "resolved tensor byte size overflows"); + } + tensor->byte_size = elements * item_size; + if (tensor->byte_size > max_tensor_size) { + lw_set_error(error, LW_STATUS_MEMORY_LIMIT, "resolved tensor exceeds max_tensor_size"); + return LW_STATUS_MEMORY_LIMIT; + } + memset(tensor->dimensions, 0, sizeof(tensor->dimensions)); + if (rank != 0u) { + memcpy(tensor->dimensions, dimensions, (size_t)rank * sizeof(dimensions[0])); + } + return LW_STATUS_OK; +} + +static int normalize_axis(int32_t axis, uint32_t rank, uint32_t* normalized) { + int64_t value = axis; + if (value < 0) { + value += rank; + } + if (value < 0 || value >= rank) { + return 0; + } + *normalized = (uint32_t)value; + return 1; +} + +static int slice_bounds(int32_t dimension, int32_t start, int32_t end, int32_t step, + int32_t* length) { + int64_t normalized_start = start; + int64_t normalized_end = end; + int64_t count; + if (dimension <= 0 || step <= 0) { + return 0; + } + if (normalized_start == INT32_MIN) { + normalized_start = 0; + } else if (normalized_start < 0) { + normalized_start += dimension; + } + if (normalized_end == INT32_MAX) { + normalized_end = dimension; + } else if (normalized_end < 0) { + normalized_end += dimension; + } + if (normalized_start < 0) normalized_start = 0; + if (normalized_start > dimension) normalized_start = dimension; + if (normalized_end < 0) normalized_end = 0; + if (normalized_end > dimension) normalized_end = dimension; + count = normalized_end <= normalized_start + ? 0 + : (normalized_end - normalized_start + step - 1) / step; + if (count <= 0 || count > INT32_MAX) return 0; + *length = (int32_t)count; + return 1; +} + +static int broadcast_dimension(int32_t left, int32_t right, int32_t* output) { + if (left == right) { + *output = left; + return 1; + } + if (left == 1) { + *output = right; + return 1; + } + if (right == 1) { + *output = left; + return 1; + } + return 0; +} + +static lw_status broadcast_shape(const lw_runtime_tensor* left, const lw_runtime_tensor* right, + uint32_t* output_rank, int32_t* output_dimensions, + lw_error* error) { + uint32_t rank = left->rank > right->rank ? left->rank : right->rank; + uint32_t i; + if (left->dtype != right->dtype) { + return shape_fail(error, "elementwise input dtypes do not match"); + } + for (i = 0u; i < rank; ++i) { + int32_t left_dim = i < rank - left->rank ? 1 : left->dimensions[i - (rank - left->rank)]; + int32_t right_dim = + i < rank - right->rank ? 1 : right->dimensions[i - (rank - right->rank)]; + if (!broadcast_dimension(left_dim, right_dim, &output_dimensions[i])) { + return shape_fail(error, "elementwise input shapes cannot be broadcast"); + } + } + *output_rank = rank; + return LW_STATUS_OK; +} + +static int spatial_output(int32_t input, int32_t kernel, int32_t stride, int32_t dilation, + int32_t pad_before, int32_t pad_after, uint32_t ceil_mode, + int32_t* output) { + int64_t effective; + int64_t numerator; + int64_t value; + if (input <= 0 || kernel <= 0 || stride <= 0 || dilation <= 0 || pad_before < 0 || + pad_after < 0) { + return 0; + } + effective = (int64_t)(kernel - 1) * dilation + 1; + numerator = (int64_t)input + pad_before + pad_after - effective; + if (numerator < 0) { + return 0; + } + value = (numerator + (ceil_mode != 0u ? stride - 1 : 0)) / stride + 1; + if (ceil_mode != 0u && (value - 1) * stride >= (int64_t)input + pad_before) { + --value; + } + if (value <= 0 || value > INT32_MAX) { + return 0; + } + *output = (int32_t)value; + return 1; +} + +static int transposed_spatial_output(int32_t input, int32_t kernel, int32_t stride, + int32_t dilation, int32_t pad_before, int32_t pad_after, + int32_t* output) { + int64_t value; + if (input <= 0 || kernel <= 0 || stride <= 0 || dilation <= 0 || pad_before < 0 || + pad_after < 0) { + return 0; + } + value = (int64_t)(input - 1) * stride - pad_before - pad_after + + (int64_t)(kernel - 1) * dilation + 1; + if (value <= 0 || value > INT32_MAX) { + return 0; + } + *output = (int32_t)value; + return 1; +} + +lw_status lw_resolve_import_node(lw_session* session, const uint8_t* node, + uint64_t max_tensor_size, lw_error* error) { + const lw_model* model = session->model; + uint16_t op = lwm_read_u16(node); + uint16_t input_count = lwm_read_u16(node + 2); + uint16_t output_count = lwm_read_u16(node + 4); + uint64_t param_offset = lwm_read_u64(node + 56); + const uint8_t* params = param_offset == 0u ? NULL : model->bytes + (size_t)param_offset; + const lw_runtime_tensor* inputs[LWM_V0_MAX_NODE_INPUTS]; + lw_runtime_tensor* output; + int32_t dimensions[LW_MAX_DIMS] = {0}; + uint32_t rank = 0u; + uint32_t i; + lw_status status; + + if (output_count != 1u) { + return shape_fail(error, "current REC shape resolver requires one output per node"); + } + for (i = 0u; i < input_count; ++i) { + inputs[i] = &session->tensors[lwm_read_u32(node + 8u + i * 4u)]; + } + output = &session->tensors[lwm_read_u32(node + 40)]; + + if (op == 2u || op == 3u || op == 4u || op == 22u || op == 24u) { + if (input_count != 2u) { + return shape_fail(error, "elementwise node has invalid arity"); + } + status = broadcast_shape(inputs[0], inputs[1], &rank, dimensions, error); + if (status != LW_STATUS_OK) { + return status; + } + } else if (op == 5u || op == 6u || op == 9u || op == 15u || op == 21u || op == 23u) { + if (input_count != 1u) { + return shape_fail(error, "unary node has invalid arity"); + } + rank = inputs[0]->rank; + memcpy(dimensions, inputs[0]->dimensions, sizeof(dimensions)); + if (op == 15u) { + uint32_t axis; + if (!normalize_axis(lwm_read_i32(params + 4), rank, &axis)) { + return shape_fail(error, "Softmax axis is outside the input rank"); + } + } + } else if (op == 25u) { + uint32_t slice_count = lwm_read_u16(params + 2u); + int selected[LW_MAX_DIMS] = {0}; + if (input_count != 1u || slice_count == 0u || slice_count > inputs[0]->rank) { + return shape_fail(error, "Slice has invalid arity or count"); + } + rank = inputs[0]->rank; + memcpy(dimensions, inputs[0]->dimensions, sizeof(dimensions)); + for (i = 0u; i < slice_count; ++i) { + uint32_t axis; + if (!normalize_axis(lwm_read_i32(params + 68u + i * 4u), rank, &axis) || + selected[axis] || !slice_bounds(inputs[0]->dimensions[axis], + lwm_read_i32(params + 4u + i * 4u), + lwm_read_i32(params + 36u + i * 4u), + lwm_read_i32(params + 100u + i * 4u), + &dimensions[axis])) { + return shape_fail(error, "Slice bounds are invalid"); + } + selected[axis] = 1; + } + } else if (op == 1u) { + uint32_t group; + if ((input_count != 2u && input_count != 3u) || inputs[0]->rank != 4u || + inputs[1]->rank != 4u) { + return shape_fail(error, "Conv requires NCHW input and OIHW weight"); + } + if (inputs[0]->dtype != inputs[1]->dtype || + (input_count == 3u && (inputs[2]->dtype != inputs[0]->dtype || inputs[2]->rank != 1u || + inputs[2]->dimensions[0] != inputs[1]->dimensions[0]))) { + return shape_fail(error, "Conv weight or bias type/shape is inconsistent"); + } + group = lwm_read_u32(params + 4); + if ((uint64_t)inputs[1]->dimensions[1] * group != (uint32_t)inputs[0]->dimensions[1] || + inputs[1]->dimensions[2] != lwm_read_i32(params + 8) || + inputs[1]->dimensions[3] != lwm_read_i32(params + 12)) { + return shape_fail(error, "Conv channel or kernel dimensions are inconsistent"); + } + rank = 4u; + dimensions[0] = inputs[0]->dimensions[0]; + dimensions[1] = inputs[1]->dimensions[0]; + if (!spatial_output(inputs[0]->dimensions[2], lwm_read_i32(params + 8), + lwm_read_i32(params + 16), lwm_read_i32(params + 24), + lwm_read_i32(params + 32), lwm_read_i32(params + 40), 0u, + &dimensions[2]) || + !spatial_output(inputs[0]->dimensions[3], lwm_read_i32(params + 12), + lwm_read_i32(params + 20), lwm_read_i32(params + 28), + lwm_read_i32(params + 36), lwm_read_i32(params + 44), 0u, + &dimensions[3])) { + return shape_fail(error, "Conv output shape is invalid"); + } + } else if (op == 7u) { + if (input_count != 5u || inputs[0]->rank < 2u) { + return shape_fail(error, "BatchNormalization has invalid input arity or rank"); + } + for (i = 1u; i < 5u; ++i) { + if (inputs[i]->dtype != inputs[0]->dtype || inputs[i]->rank != 1u || + inputs[i]->dimensions[0] != inputs[0]->dimensions[1]) { + return shape_fail(error, "BatchNormalization parameter shape is invalid"); + } + } + rank = inputs[0]->rank; + memcpy(dimensions, inputs[0]->dimensions, sizeof(dimensions)); + } else if (op == 8u) { + uint32_t axes_count; + uint32_t keepdims; + int reduced[LW_MAX_DIMS] = {0}; + uint32_t output_index = 0u; + if (input_count != 1u) { + return shape_fail(error, "ReduceMean has invalid arity"); + } + axes_count = lwm_read_u16(params + 2); + keepdims = lwm_read_u32(params + 4); + if (axes_count == 0u && lwm_read_u32(params + 8) != 0u) { + rank = inputs[0]->rank; + memcpy(dimensions, inputs[0]->dimensions, sizeof(dimensions)); + } else if (axes_count == 0u) { + for (i = 0u; i < inputs[0]->rank; ++i) { + reduced[i] = 1; + } + } else { + for (i = 0u; i < axes_count; ++i) { + uint32_t axis; + if (!normalize_axis(lwm_read_i32(params + 12u + i * 4u), inputs[0]->rank, &axis) || + reduced[axis]) { + return shape_fail(error, "ReduceMean axes are invalid or duplicated"); + } + reduced[axis] = 1; + } + } + for (i = 0u; i < inputs[0]->rank; ++i) { + if (reduced[i]) { + if (keepdims != 0u) { + dimensions[output_index++] = 1; + } + } else { + dimensions[output_index++] = inputs[0]->dimensions[i]; + } + } + rank = output_index; + } else if (op == 10u || op == 19u) { + if (input_count != 1u || inputs[0]->rank != 4u) { + return shape_fail(error, "pool operator requires one NCHW input"); + } + rank = 4u; + dimensions[0] = inputs[0]->dimensions[0]; + dimensions[1] = inputs[0]->dimensions[1]; + if (!spatial_output(inputs[0]->dimensions[2], lwm_read_i32(params + 8), + lwm_read_i32(params + 16), 1, lwm_read_i32(params + 24), + lwm_read_i32(params + 32), lwm_read_u32(params + 40), &dimensions[2]) || + !spatial_output(inputs[0]->dimensions[3], lwm_read_i32(params + 12), + lwm_read_i32(params + 20), 1, lwm_read_i32(params + 28), + lwm_read_i32(params + 36), lwm_read_u32(params + 40), &dimensions[3])) { + return shape_fail(error, "pool output shape is invalid"); + } + } else if (op == 11u) { + uint32_t axes_count; + int squeezed[LW_MAX_DIMS] = {0}; + uint32_t output_index = 0u; + if (input_count != 1u) { + return shape_fail(error, "Squeeze has invalid arity"); + } + axes_count = lwm_read_u16(params + 2); + if (axes_count == 0u) { + for (i = 0u; i < inputs[0]->rank; ++i) { + squeezed[i] = inputs[0]->dimensions[i] == 1; + } + } else { + for (i = 0u; i < axes_count; ++i) { + uint32_t axis; + if (!normalize_axis(lwm_read_i32(params + 4u + i * 4u), inputs[0]->rank, &axis) || + squeezed[axis] || inputs[0]->dimensions[axis] != 1) { + return shape_fail( + error, "Squeeze axes are invalid, duplicated, or not unit dimensions"); + } + squeezed[axis] = 1; + } + } + for (i = 0u; i < inputs[0]->rank; ++i) { + if (!squeezed[i]) { + dimensions[output_index++] = inputs[0]->dimensions[i]; + } + } + rank = output_index; + } else if (op == 13u) { + uint32_t axes_count; + uint32_t output_rank; + int inserted[LW_MAX_DIMS] = {0}; + uint32_t input_index = 0u; + if (input_count != 1u) { + return shape_fail(error, "Unsqueeze has invalid arity"); + } + axes_count = lwm_read_u16(params + 2); + output_rank = inputs[0]->rank + axes_count; + if (output_rank > LW_MAX_DIMS) { + return shape_fail(error, "Unsqueeze output rank exceeds the runtime limit"); + } + for (i = 0u; i < axes_count; ++i) { + uint32_t axis; + if (!normalize_axis(lwm_read_i32(params + 4u + i * 4u), output_rank, &axis) || + inserted[axis]) { + return shape_fail(error, "Unsqueeze axes are invalid or duplicated"); + } + inserted[axis] = 1; + } + for (i = 0u; i < output_rank; ++i) { + dimensions[i] = inserted[i] ? 1 : inputs[0]->dimensions[input_index++]; + } + rank = output_rank; + } else if (op == 12u) { + uint32_t perm_count; + int used[LW_MAX_DIMS] = {0}; + if (input_count != 1u) { + return shape_fail(error, "Transpose has invalid arity"); + } + perm_count = lwm_read_u16(params + 2); + if (perm_count != inputs[0]->rank) { + return shape_fail(error, "Transpose perm length disagrees with input rank"); + } + rank = inputs[0]->rank; + for (i = 0u; i < rank; ++i) { + int32_t axis_value = lwm_read_i32(params + 4u + i * 4u); + if (axis_value < 0 || (uint32_t)axis_value >= rank || used[axis_value]) { + return shape_fail(error, "Transpose perm is invalid or duplicated"); + } + used[axis_value] = 1; + dimensions[i] = inputs[0]->dimensions[axis_value]; + } + } else if (op == 14u) { + uint32_t batch_rank; + uint32_t left_offset; + uint32_t right_offset; + if (input_count != 2u || inputs[0]->dtype != inputs[1]->dtype || inputs[0]->rank < 2u || + inputs[1]->rank < 2u) { + return shape_fail(error, "MatMul currently requires inputs with rank at least two"); + } + if (inputs[0]->dimensions[inputs[0]->rank - 1u] != + inputs[1]->dimensions[inputs[1]->rank - 2u]) { + return shape_fail(error, "MatMul inner dimensions do not match"); + } + batch_rank = (inputs[0]->rank - 2u) > (inputs[1]->rank - 2u) ? inputs[0]->rank - 2u + : inputs[1]->rank - 2u; + rank = batch_rank + 2u; + left_offset = batch_rank - (inputs[0]->rank - 2u); + right_offset = batch_rank - (inputs[1]->rank - 2u); + for (i = 0u; i < batch_rank; ++i) { + int32_t left_dim = i < left_offset ? 1 : inputs[0]->dimensions[i - left_offset]; + int32_t right_dim = i < right_offset ? 1 : inputs[1]->dimensions[i - right_offset]; + if (!broadcast_dimension(left_dim, right_dim, &dimensions[i])) { + return shape_fail(error, "MatMul batch dimensions cannot be broadcast"); + } + } + dimensions[batch_rank] = inputs[0]->dimensions[inputs[0]->rank - 2u]; + dimensions[batch_rank + 1u] = inputs[1]->dimensions[inputs[1]->rank - 1u]; + } else if (op == 16u) { + uint64_t input_elements = 1u; + uint64_t output_elements = 1u; + int32_t unknown_dimension = -1; + if (input_count != 1u || output->rank > LW_MAX_DIMS) { + return shape_fail(error, "Reshape has invalid arity or output rank"); + } + rank = output->rank; + memcpy(dimensions, output->dimensions, sizeof(dimensions)); + for (i = 0u; i < inputs[0]->rank; ++i) { + if (inputs[0]->dimensions[i] <= 0 || + input_elements > UINT64_MAX / (uint32_t)inputs[0]->dimensions[i]) { + return shape_fail(error, "Reshape input element count is invalid"); + } + input_elements *= (uint32_t)inputs[0]->dimensions[i]; + } + for (i = 0u; i < rank; ++i) { + if (dimensions[i] == -1) { + if (unknown_dimension != -1) { + return shape_fail(error, "Reshape has more than one unresolved dimension"); + } + unknown_dimension = (int32_t)i; + continue; + } + if (dimensions[i] <= 0 || output_elements > UINT64_MAX / (uint32_t)dimensions[i]) { + return shape_fail(error, "Reshape output element count is invalid"); + } + output_elements *= (uint32_t)dimensions[i]; + } + if (unknown_dimension != -1) { + if (output_elements == 0u || input_elements % output_elements != 0u || + input_elements / output_elements > INT32_MAX) { + return shape_fail(error, "Reshape unresolved dimension is invalid"); + } + dimensions[unknown_dimension] = (int32_t)(input_elements / output_elements); + output_elements = input_elements; + } + if (input_elements != output_elements) { + return shape_fail(error, "Reshape changes the tensor element count"); + } + } else if (op == 17u) { + uint32_t axis; + int64_t axis_total = 0; + if (input_count == 0u || + !normalize_axis(lwm_read_i32(params + 4), inputs[0]->rank, &axis)) { + return shape_fail(error, "Concat has invalid arity or axis"); + } + rank = inputs[0]->rank; + memcpy(dimensions, inputs[0]->dimensions, sizeof(dimensions)); + for (i = 0u; i < input_count; ++i) { + uint32_t dimension; + if (inputs[i]->dtype != inputs[0]->dtype || inputs[i]->rank != rank) { + return shape_fail(error, "Concat input types or ranks do not match"); + } + for (dimension = 0u; dimension < rank; ++dimension) { + if (dimension != axis && + inputs[i]->dimensions[dimension] != dimensions[dimension]) { + return shape_fail(error, "Concat non-axis dimensions do not match"); + } + } + axis_total += inputs[i]->dimensions[axis]; + } + if (axis_total <= 0 || axis_total > INT32_MAX) { + return shape_fail(error, "Concat axis dimension overflows"); + } + dimensions[axis] = (int32_t)axis_total; + } else if (op == 18u) { + uint32_t group; + uint64_t output_channels; + if ((input_count != 2u && input_count != 3u) || inputs[0]->rank != 4u || + inputs[1]->rank != 4u || inputs[0]->dtype != inputs[1]->dtype) { + return shape_fail(error, "ConvTranspose requires NCHW input and IOHW weight"); + } + group = lwm_read_u32(params + 4); + output_channels = (uint64_t)(uint32_t)inputs[1]->dimensions[1] * group; + rank = 4u; + dimensions[0] = inputs[0]->dimensions[0]; + if (group == 0u || group > (uint32_t)inputs[0]->dimensions[1] || + output_channels > INT32_MAX || (uint32_t)inputs[0]->dimensions[1] % group != 0u || + inputs[1]->dimensions[0] != inputs[0]->dimensions[1] || + inputs[1]->dimensions[2] != lwm_read_i32(params + 8) || + inputs[1]->dimensions[3] != lwm_read_i32(params + 12) || + (input_count == 3u && (inputs[2]->dtype != inputs[0]->dtype || inputs[2]->rank != 1u || + inputs[2]->dimensions[0] != (int32_t)output_channels)) || + !transposed_spatial_output(inputs[0]->dimensions[2], lwm_read_i32(params + 8), + lwm_read_i32(params + 16), lwm_read_i32(params + 24), + lwm_read_i32(params + 32), lwm_read_i32(params + 40), + &dimensions[2]) || + !transposed_spatial_output(inputs[0]->dimensions[3], lwm_read_i32(params + 12), + lwm_read_i32(params + 20), lwm_read_i32(params + 28), + lwm_read_i32(params + 36), lwm_read_i32(params + 44), + &dimensions[3])) { + return shape_fail(error, "ConvTranspose channel, kernel, or output shape is invalid"); + } + dimensions[1] = (int32_t)output_channels; + } else if (op == 20u) { + uint32_t scale_count = lwm_read_u16(params + 2); + if (input_count != 1u || scale_count != inputs[0]->rank) { + return shape_fail(error, "Resize scale count does not match the input rank"); + } + rank = inputs[0]->rank; + for (i = 0u; i < rank; ++i) { + float scale = read_f32(params + 4u + i * 4u); + double value = floor((double)inputs[0]->dimensions[i] * scale); + if (!isfinite(scale) || scale <= 0.0f || value <= 0.0 || value > INT32_MAX) { + return shape_fail(error, "Resize scale produces an invalid output dimension"); + } + dimensions[i] = (int32_t)value; + } + } else { + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "shape resolver encountered an unsupported operator"); + return LW_STATUS_UNSUPPORTED; + } + return set_tensor_shape(output, inputs[0]->dtype, rank, dimensions, max_tensor_size, error); +} + +lw_status lw_resolve_shapes(lw_session* session, uint64_t max_tensor_size, lw_error* error) { + const lw_model* model = session->model; + uint32_t i; + for (i = 0u; i < model->info.node_count; ++i) { + const uint8_t* node = + model->bytes + (size_t)model->node_offset + (size_t)i * LWM_V0_NODE_SIZE; + lw_status status = lw_resolve_import_node(session, node, max_tensor_size, error); + if (status != LW_STATUS_OK) { + return status; + } + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/validate.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/validate.c new file mode 100644 index 00000000..e792d233 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/validate.c @@ -0,0 +1,455 @@ +#include "model_internal.h" + +/* + * Defensive parser for the on-disk LWM v0.1 format. + * Treat every offset, count, shape and operator parameter as untrusted input. + * Checks are deliberately performed before pointer arithmetic to avoid an + * integer wrap turning malformed model data into an out-of-bounds access. + */ + +#include +#include +#include +#include + +typedef struct lwm_header { + uint32_t flags; + uint32_t tensor_count; + uint32_t node_count; + uint32_t input_count; + uint32_t output_count; + uint64_t input_offset; + uint64_t output_offset; + uint64_t tensor_offset; + uint64_t node_offset; + uint64_t param_offset; + uint64_t param_size; + uint64_t string_offset; + uint64_t string_size; + uint64_t weight_offset; + uint64_t weight_size; + uint64_t file_size; + uint64_t workspace_size; + uint64_t checksum; +} lwm_header; + +static uint16_t read_u16(const uint8_t* p) { + return (uint16_t)((uint16_t)p[0] | ((uint16_t)p[1] << 8)); +} + +static uint32_t read_u32(const uint8_t* p) { + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} + +static int32_t read_i32(const uint8_t* p) { + return (int32_t)read_u32(p); +} + +static float read_f32(const uint8_t* p) { + uint32_t bits = read_u32(p); + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static uint64_t read_u64(const uint8_t* p) { + return (uint64_t)read_u32(p) | ((uint64_t)read_u32(p + 4) << 32); +} + +static int is_aligned8(uint64_t value) { + return (value & UINT64_C(7)) == 0u; +} + +static int range_valid(uint64_t offset, uint64_t size, uint64_t file_size) { + return offset <= file_size && size <= file_size - offset; +} + +static int table_range_valid(uint64_t offset, uint32_t count, uint32_t record_size, + uint64_t file_size) { + uint64_t size = (uint64_t)count * (uint64_t)record_size; + return range_valid(offset, size, file_size); +} + +static uint64_t table_end(uint64_t offset, uint32_t count, uint32_t record_size) { + return offset + (uint64_t)count * (uint64_t)record_size; +} + +static lw_status fail(lw_error* error, lw_status status, const char* message) { + lw_set_error(error, status, message); + return status; +} + +static uint32_t dtype_size(uint32_t dtype) { + switch (dtype) { + case 1u: + return 4u; + case 2u: + return 4u; + case 3u: + return 8u; + case 4u: + return 1u; + default: + return 0u; + } +} + +static uint32_t expected_param_size(uint16_t op) { + switch (op) { + case 1u: + return 64u; + case 6u: + return 16u; + case 7u: + return 24u; + case 8u: + return 48u; + case 10u: + return 64u; + case 11u: + return 40u; + case 12u: + return 40u; + case 13u: + return 40u; + case 15u: + return 16u; + case 17u: + return 16u; + case 18u: + return 64u; + case 19u: + return 64u; + case 20u: + return 32u; + case 25u: + return 136u; + default: + return 0u; + } +} + +static int param_in_section(uint64_t offset, uint32_t size, const lwm_header* h) { + return offset >= h->param_offset && range_valid(offset, size, h->param_offset + h->param_size); +} + +static lw_status validate_params(const uint8_t* p, uint16_t op, uint32_t size, lw_error* error) { + uint32_t i; + if (size == 0u) { + return LW_STATUS_OK; + } + if (read_u16(p) != 1u) { + return fail(error, LW_STATUS_UNSUPPORTED_VERSION, "unsupported operator parameter version"); + } + if (op == 1u || op == 18u) { + if (read_u16(p + 2) != 2u || read_u32(p + 4) == 0u || read_u32(p + 4) > INT32_MAX) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid convolution parameter record"); + } + for (i = 0u; i < 6u; ++i) { + if (read_i32(p + 8u + i * 4u) <= 0) { + return fail(error, LW_STATUS_INVALID_FORMAT, + "convolution kernel, stride, or dilation is invalid"); + } + } + for (i = 0u; i < 4u; ++i) { + if (read_i32(p + 32u + i * 4u) < 0 || read_u32(p + 48u + i * 4u) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, + "convolution padding or reserved field is invalid"); + } + } + } else if (op == 6u) { + if (read_u16(p + 2) != 0u || read_u32(p + 12) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid HardSigmoid parameter record"); + } + } else if (op == 7u) { + if (read_u16(p + 2) != 0u || read_u32(p + 12) != 0u || read_u32(p + 16) != 0u || + read_u32(p + 20) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, + "invalid BatchNormalization parameter record"); + } + } else if (op == 8u) { + if (read_u16(p + 2) > LWM_V0_MAX_DIMS || read_u32(p + 4) > 1u || read_u32(p + 8) > 1u || + read_u32(p + 44) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid ReduceMean parameter record"); + } + } else if (op == 10u || op == 19u) { + if (read_u16(p + 2) != 0u || read_u32(p + 4) != 2u || read_u32(p + 40) > 1u || + read_u32(p + 44) > (op == 10u ? 1u : 0u)) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid pool parameter record"); + } + for (i = 0u; i < 4u; ++i) { + if (read_i32(p + 8u + i * 4u) <= 0) { + return fail(error, LW_STATUS_INVALID_FORMAT, "pool kernel or stride is invalid"); + } + } + for (i = 0u; i < 4u; ++i) { + if (read_i32(p + 24u + i * 4u) < 0) { + return fail(error, LW_STATUS_INVALID_FORMAT, "pool padding is invalid"); + } + } + for (i = 0u; i < 4u; ++i) { + if (read_u32(p + 48u + i * 4u) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "pool reserved field is non-zero"); + } + } + } else if (op == 11u || op == 12u || op == 13u) { + if (read_u16(p + 2) > LWM_V0_MAX_DIMS || read_u32(p + 36) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid axes parameter record"); + } + } else if (op == 15u) { + if (read_u16(p + 2) != 0u || read_u32(p + 8) != 0u || read_u32(p + 12) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid Softmax parameter record"); + } + } else if (op == 17u) { + if (read_u16(p + 2) != 0u || read_u32(p + 8) != 0u || read_u32(p + 12) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid Concat parameter record"); + } + } else if (op == 20u) { + if (read_u16(p + 2) != 4u || read_u32(p + 20) != 0u || read_u32(p + 24) != 0u || + read_u32(p + 28) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid Resize parameter record"); + } + for (i = 0u; i < 4u; ++i) { + float scale = read_f32(p + 4u + i * 4u); + if (!isfinite(scale) || scale <= 0.0f) { + return fail(error, LW_STATUS_INVALID_FORMAT, "Resize scale is invalid"); + } + } + } else if (op == 25u) { + if (read_u16(p + 2u) == 0u || read_u16(p + 2u) > LWM_V0_MAX_DIMS || + read_u32(p + 132u) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid Slice parameter record"); + } + for (i = 0u; i < read_u16(p + 2u); ++i) { + if (read_i32(p + 100u + i * 4u) <= 0) { + return fail(error, LW_STATUS_INVALID_FORMAT, "Slice steps must be positive"); + } + } + } + return LW_STATUS_OK; +} + +static uint64_t compute_checksum(const uint8_t* bytes, size_t count) { + uint64_t value = UINT64_C(14695981039346656037); + size_t i; + for (i = 0u; i < count; ++i) { + uint8_t byte = + (i >= LWM_V0_CHECKSUM_OFFSET && i < LWM_V0_CHECKSUM_OFFSET + 8u) ? 0u : bytes[i]; + value ^= byte; + value *= UINT64_C(1099511628211); + } + return value; +} + +lw_status lw_validate_lwm_v0(lw_model* model, lw_error* error) { + const uint8_t* b = model->bytes; + lwm_header h; + uint32_t i; + /* Validate the fixed header and section map before inspecting any table. */ + if (memcmp(b, "LWM0", 4u) != 0) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid LWM magic"); + } + if (read_u16(b + 4) != 0u || read_u16(b + 6) != 1u) { + return fail(error, LW_STATUS_UNSUPPORTED_VERSION, "unsupported LWM format version"); + } + if (read_u32(b + 8) != LWM_V0_HEADER_SIZE) { + return fail(error, LW_STATUS_INVALID_FORMAT, "invalid LWM header size"); + } + h.flags = read_u32(b + 12); + h.tensor_count = read_u32(b + 16); + h.node_count = read_u32(b + 20); + h.input_count = read_u32(b + 24); + h.output_count = read_u32(b + 28); + h.input_offset = read_u64(b + 32); + h.output_offset = read_u64(b + 40); + h.tensor_offset = read_u64(b + 48); + h.node_offset = read_u64(b + 56); + h.param_offset = read_u64(b + 64); + h.param_size = read_u64(b + 72); + h.string_offset = read_u64(b + 80); + h.string_size = read_u64(b + 88); + h.weight_offset = read_u64(b + 96); + h.weight_size = read_u64(b + 104); + h.file_size = read_u64(b + 112); + h.workspace_size = read_u64(b + 120); + h.checksum = read_u64(b + 128); + + if (h.flags != LWM_V0_HEADER_FLAG_NO_MEMORY_PLAN || h.workspace_size != 0u) { + return fail(error, LW_STATUS_UNSUPPORTED, "unsupported LWM header flags or workspace plan"); + } + if (read_u64(b + 136) != 0u || read_u64(b + 144) != 0u || read_u64(b + 152) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "non-zero LWM header reserved field"); + } + if (h.file_size != model->byte_count) { + return fail(error, LW_STATUS_OUT_OF_BOUNDS, "LWM file_size does not match the actual file"); + } + if (!is_aligned8(h.input_offset) || !is_aligned8(h.output_offset) || + !is_aligned8(h.tensor_offset) || !is_aligned8(h.node_offset) || + !is_aligned8(h.param_offset) || !is_aligned8(h.string_offset) || + !is_aligned8(h.weight_offset)) { + return fail(error, LW_STATUS_INVALID_FORMAT, "LWM section offset is not 8-byte aligned"); + } + if (!table_range_valid(h.input_offset, h.input_count, 4u, h.file_size) || + !table_range_valid(h.output_offset, h.output_count, 4u, h.file_size) || + !table_range_valid(h.tensor_offset, h.tensor_count, LWM_V0_TENSOR_SIZE, h.file_size) || + !table_range_valid(h.node_offset, h.node_count, LWM_V0_NODE_SIZE, h.file_size) || + !range_valid(h.param_offset, h.param_size, h.file_size) || + !range_valid(h.string_offset, h.string_size, h.file_size) || + !range_valid(h.weight_offset, h.weight_size, h.file_size)) { + return fail(error, LW_STATUS_OUT_OF_BOUNDS, "LWM section range exceeds file bounds"); + } + if (h.input_offset < LWM_V0_HEADER_SIZE || + table_end(h.input_offset, h.input_count, 4u) > h.output_offset || + table_end(h.output_offset, h.output_count, 4u) > h.tensor_offset || + table_end(h.tensor_offset, h.tensor_count, LWM_V0_TENSOR_SIZE) > h.node_offset || + table_end(h.node_offset, h.node_count, LWM_V0_NODE_SIZE) > h.param_offset || + h.param_offset + h.param_size > h.string_offset || + h.string_offset + h.string_size > h.weight_offset || + h.weight_offset + h.weight_size != h.file_size) { + return fail(error, LW_STATUS_OUT_OF_BOUNDS, "LWM sections overlap or are out of order"); + } + if (h.checksum == 0u || compute_checksum(b, model->byte_count) != h.checksum) { + return fail(error, LW_STATUS_CHECKSUM_MISMATCH, "LWM content checksum mismatch"); + } + + /* Tensor records must describe either validated constant payloads or + * runtime-owned storage, never arbitrary file/workspace pointers. */ + for (i = 0u; i < h.tensor_count; ++i) { + const uint8_t* t = b + (size_t)h.tensor_offset + (size_t)i * LWM_V0_TENSOR_SIZE; + uint32_t dtype = read_u32(t); + uint32_t rank = read_u32(t + 4); + uint32_t flags = read_u32(t + 40); + uint64_t data_offset = read_u64(t + 48); + uint64_t data_size = read_u64(t + 56); + uint64_t workspace_offset = read_u64(t + 64); + uint64_t workspace_size = read_u64(t + 72); + uint64_t element_count = 1u; + uint32_t j; + int dynamic = 0; + uint32_t item_size = dtype_size(dtype); + if (item_size == 0u || rank > LWM_V0_MAX_DIMS || (flags & ~7u) != 0u || + read_u32(t + 44) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, + "invalid tensor type, rank, flags, or reserved field"); + } + for (j = 0u; j < LWM_V0_MAX_DIMS; ++j) { + int32_t dim = read_i32(t + 8u + j * 4u); + if (j < rank) { + if (dim == -1) { + dynamic = 1; + } else if (dim <= 0 || element_count > UINT64_MAX / (uint32_t)dim) { + return fail(error, LW_STATUS_OUT_OF_BOUNDS, + "invalid or overflowing tensor dimensions"); + } else { + element_count *= (uint32_t)dim; + } + } else if (dim != 0) { + return fail(error, LW_STATUS_INVALID_FORMAT, "unused tensor dimension is non-zero"); + } + } + if ((flags & LWM_V0_TENSOR_FLAG_CONSTANT) != 0u) { + if (dynamic || !is_aligned8(data_offset) || data_offset < h.weight_offset || + !range_valid(data_offset, data_size, h.weight_offset + h.weight_size) || + element_count > UINT64_MAX / item_size || data_size != element_count * item_size) { + return fail(error, LW_STATUS_OUT_OF_BOUNDS, + "invalid constant tensor data range or size"); + } + } else if (data_offset != 0u || data_size != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "non-constant tensor contains file data"); + } + if (workspace_offset != LWM_V0_NO_WORKSPACE || workspace_size != 0u) { + return fail(error, LW_STATUS_UNSUPPORTED, + "v0.1 tensor contains an unsupported workspace plan"); + } + } + + /* Node arity, tensor indices and parameter sizes are checked against the + * exact operator subset understood by executor.c. */ + for (i = 0u; i < h.node_count; ++i) { + const uint8_t* n = b + (size_t)h.node_offset + (size_t)i * LWM_V0_NODE_SIZE; + uint16_t op = read_u16(n); + uint16_t input_count = read_u16(n + 2); + uint16_t output_count = read_u16(n + 4); + uint64_t param_offset = read_u64(n + 56); + uint32_t param_size = read_u32(n + 64); + uint32_t expected_size; + uint32_t j; + lw_status param_status; + if (op == 0u || op > 25u || input_count > LWM_V0_MAX_NODE_INPUTS || output_count == 0u || + output_count > LWM_V0_MAX_NODE_OUTPUTS || read_u16(n + 6) != 0u || + read_u32(n + 68) != 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, + "invalid node operator, arity, flags, or reserved field"); + } + for (j = 0u; j < LWM_V0_MAX_NODE_INPUTS; ++j) { + uint32_t index = read_u32(n + 8u + j * 4u); + if ((j < input_count && index >= h.tensor_count) || (j >= input_count && index != 0u)) { + return fail(error, LW_STATUS_OUT_OF_BOUNDS, "invalid node input tensor index"); + } + } + for (j = 0u; j < LWM_V0_MAX_NODE_OUTPUTS; ++j) { + uint32_t index = read_u32(n + 40u + j * 4u); + if ((j < output_count && index >= h.tensor_count) || + (j >= output_count && index != 0u)) { + return fail(error, LW_STATUS_OUT_OF_BOUNDS, "invalid node output tensor index"); + } + } + expected_size = expected_param_size(op); + if (param_size != expected_size || (param_size == 0u && param_offset != 0u) || + (param_size != 0u && + (!is_aligned8(param_offset) || !param_in_section(param_offset, param_size, &h)))) { + return fail(error, LW_STATUS_OUT_OF_BOUNDS, "invalid node parameter range or size"); + } + param_status = validate_params(param_size == 0u ? NULL : b + (size_t)param_offset, op, + param_size, error); + if (param_status != LW_STATUS_OK) { + return param_status; + } + } + + for (i = 0u; i < h.input_count; ++i) { + uint32_t index = read_u32(b + (size_t)h.input_offset + (size_t)i * 4u); + const uint8_t* tensor; + if (index >= h.tensor_count) { + return fail(error, LW_STATUS_OUT_OF_BOUNDS, "invalid graph input tensor index"); + } + tensor = b + (size_t)h.tensor_offset + (size_t)index * LWM_V0_TENSOR_SIZE; + if ((read_u32(tensor + 40) & LWM_V0_TENSOR_FLAG_INPUT) == 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "graph input tensor flag is missing"); + } + } + for (i = 0u; i < h.output_count; ++i) { + uint32_t index = read_u32(b + (size_t)h.output_offset + (size_t)i * 4u); + const uint8_t* tensor; + if (index >= h.tensor_count) { + return fail(error, LW_STATUS_OUT_OF_BOUNDS, "invalid graph output tensor index"); + } + tensor = b + (size_t)h.tensor_offset + (size_t)index * LWM_V0_TENSOR_SIZE; + if ((read_u32(tensor + 40) & LWM_V0_TENSOR_FLAG_OUTPUT) == 0u) { + return fail(error, LW_STATUS_INVALID_FORMAT, "graph output tensor flag is missing"); + } + } + + /* Publish parsed offsets only after all validation passes. */ + memset(&model->info, 0, sizeof(model->info)); + model->info.struct_size = (uint32_t)sizeof(model->info); + model->info.format_major = 0u; + model->info.format_minor = 1u; + model->info.tensor_count = h.tensor_count; + model->info.node_count = h.node_count; + model->info.input_count = h.input_count; + model->info.output_count = h.output_count; + model->info.file_size = h.file_size; + model->info.weight_size = h.weight_size; + model->info.workspace_size = h.workspace_size; + model->info.content_checksum = h.checksum; + model->input_offset = h.input_offset; + model->output_offset = h.output_offset; + model->tensor_offset = h.tensor_offset; + model->node_offset = h.node_offset; + model->param_offset = h.param_offset; + model->param_size = h.param_size; + model->weight_offset = h.weight_offset; + model->weight_size = h.weight_size; + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_det_backend_compile.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_det_backend_compile.c new file mode 100644 index 00000000..64b5a0e6 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_det_backend_compile.c @@ -0,0 +1,1449 @@ +#include "x64_det_backend_internal.h" +#include "lwm_read.h" +#include "operator_internal.h" +#include "layout_planner_internal.h" +#include "../kernels/packed_conv_internal.h" + +#include +#include +#include +#include +#include +#include +#include + +static lw_status make_shape_session(const lw_model* model, uint32_t height, uint32_t width, + lw_session** out, lw_error* error) { + lw_tensor_desc input; + lw_tensor_desc_init(&input); + input.dtype = LW_DTYPE_F32; + input.rank = 4u; + input.dimensions[0] = 1; + input.dimensions[1] = 3; + input.dimensions[2] = (int32_t)height; + input.dimensions[3] = (int32_t)width; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) + return lw_session_create_metadata_only(model, &input, 1u, NULL, out, error); +#else + return lw_session_create(model, &input, 1u, NULL, out, error); +#endif +} + +static const uint8_t* node_bytes(const lw_model* model, uint32_t index) { + return model->bytes + (size_t)model->node_offset + (size_t)index * LWM_V0_NODE_SIZE; +} + +static const float* constant_f32(const lw_model* model, uint32_t index) { + const uint8_t* tensor = model->bytes + (size_t)model->tensor_offset + + (size_t)index * LWM_V0_TENSOR_SIZE; + return (const float*)(const void*)(model->bytes + (size_t)lwm_read_u64(tensor + 48u)); +} + +static float read_f32(const uint8_t* p) { + uint32_t bits = lwm_read_u32(p); + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static uint64_t value_elements(const lw_x64_det_value* value) { return value->bytes / sizeof(float); } + +/* Effective layout of a tensor's primary slot: whatever layout its producer + * node runs in. The WASM detector writes direct-NHWC graph input into its + * arena slot; treating that slot as NCHW would convert it a second time. */ +static uint8_t det_tensor_eff_layout(const lw_session* session, + const lw_x64_det_program* program, uint32_t tensor_index) { + int32_t birth; + if (session == NULL || program == NULL || tensor_index >= program->value_count) { + return LW_X64_DET_LAYOUT_NCHW; + } +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) + if (program->direct_nhwc != 0u && + (session->tensors[tensor_index].flags & LWM_V0_TENSOR_FLAG_INPUT) != 0u) { + return LW_X64_DET_LAYOUT_NHWC; + } +#endif + birth = session->tensors[tensor_index].birth_node; + if (birth < 0 || (uint32_t)birth >= program->model->info.node_count) { + return LW_X64_DET_LAYOUT_NCHW; + } + return program->effective_node_layout[birth]; +} + +static void fill_value(const lw_session* session, uint32_t i, lw_x64_det_value* value, + const lw_x64_det_program* program) { + const lw_runtime_tensor* tensor = &session->tensors[i]; + uint32_t j; + memset(value, 0, sizeof(*value)); + value->bytes = tensor->byte_size; + value->rank = tensor->rank > 4u ? 4u : tensor->rank; + value->layout = LW_X64_DET_LAYOUT_NCHW; + if (tensor->rank == 4u && det_tensor_eff_layout(session, program, i) == LW_X64_DET_LAYOUT_NHWC) { + /* Physical order NHWC: [N,H,W,C]. */ + value->dimensions[0] = tensor->dimensions[0]; + value->dimensions[1] = tensor->dimensions[2]; + value->dimensions[2] = tensor->dimensions[3]; + value->dimensions[3] = tensor->dimensions[1]; + value->layout = LW_X64_DET_LAYOUT_NHWC; + } else { + for (j = 0u; j < value->rank; ++j) value->dimensions[j] = tensor->dimensions[j]; + } + value->producer = -1; + value->last_use = -1; + value->alt_producer = -1; + value->alt_last_use = -1; + if ((tensor->flags & LWM_V0_TENSOR_FLAG_CONSTANT) != 0u) { + value->constant_data = constant_f32(session->model, i); + } +} + +static int add_constant(lw_x64_det_program* program, void* data, uint64_t bytes) { + lw_x64_det_constant* next; + if (data == NULL || bytes == 0u) return 0; + next = (lw_x64_det_constant*)realloc(program->constants, + (size_t)(program->packed_constant_count + 1u) * sizeof(*next)); + if (next == NULL) return 0; + program->constants = next; + program->constants[program->packed_constant_count].data = data; + program->constants[program->packed_constant_count].bytes = bytes; + ++program->packed_constant_count; + return 1; +} + +static void* alloc_constant(lw_x64_det_program* program, uint64_t bytes) { + void* data; + if (bytes == 0u || bytes > SIZE_MAX) return NULL; + data = malloc((size_t)bytes); + if (data == NULL || !add_constant(program, data, bytes)) { + free(data); + return NULL; + } + return data; +} + +static uint8_t choose_nchw_pointwise_kernel(uint32_t height, uint32_t input_channels, + uint32_t output_channels, uint32_t spatial) { + if (output_channels >= 8u && (output_channels & 7u) == 0u && spatial >= 8u && + ((height == 6u && input_channels == 512u && output_channels == 1024u) || + (height == 3u && ((input_channels == 768u && output_channels == 384u) || + (input_channels == 1536u && output_channels == 768u))))) { + return LW_X64_DET_NCHW_PW_FMA8; + } + return LW_X64_DET_NCHW_PW_FMA4; +} + +static uint8_t choose_pointwise_kernel(uint32_t pixels, uint32_t output_channels) { + if (pixels <= 1u && output_channels % 32u == 0u) return LW_X64_DET_PW_2X32; + if (pixels <= 1u && output_channels % 16u == 0u) return LW_X64_DET_PW_4X16; + return LW_X64_DET_PW_6X16; +} + +static int fused_gelu_temporaries_private(const lw_model* model, uint32_t node_index, + const lw_fused_gelu_match* match) { + uint32_t temporary; + if (model == NULL || match == NULL || node_index > model->info.node_count || + model->info.node_count - node_index < 5u) { + return 0; + } + for (temporary = 0u; temporary < 4u; ++temporary) { + uint32_t tensor = match->outputs[temporary]; + uint32_t node; + int32_t last_use = -1; + for (node = 0u; node < model->info.node_count; ++node) { + const uint8_t* current = node_bytes(model, node); + uint16_t input_count = lwm_read_u16(current + 2u); + uint16_t input_index; + for (input_index = 0u; input_index < input_count; ++input_index) { + if (lwm_read_u32(current + 8u + (size_t)input_index * sizeof(uint32_t)) == tensor) { + last_use = (int32_t)node; + } + } + } + for (node = 0u; node < model->info.output_count; ++node) { + if (lwm_read_u32(model->bytes + (size_t)model->output_offset + + (size_t)node * sizeof(uint32_t)) == tensor) { + last_use = (int32_t)model->info.node_count; + } + } + if (last_use != (int32_t)(node_index + temporary + 1u)) { + return 0; + } + } + return 1; +} + +static lw_x64_det_op* push_op(lw_x64_det_program* program, uint32_t* count, uint16_t kind, + uint32_t semantic_begin, uint16_t semantic_count) { + lw_x64_det_op* op = &program->ops[*count]; + memset(op, 0, sizeof(*op)); + op->kind = kind; + op->semantic_begin = semantic_begin; + op->semantic_count = semantic_count; + ++*count; + return op; +} + +static int det_conv_is_depthwise(const lw_runtime_tensor* input, const lw_runtime_tensor* weight, + uint32_t groups) { + return groups == (uint32_t)input->dimensions[1] && + groups == (uint32_t)weight->dimensions[0] && weight->dimensions[1] == 1u; +} + +static uint64_t det_slot_offset(lw_x64_det_program* program, uint32_t tensor_index, + uint8_t consumer_layout); + +static lw_status compile_conv(const lw_model* model, const lw_session* session, + lw_x64_det_program* program, const uint8_t* node, + lw_x64_det_op* op, lw_error* error) { + const lw_runtime_tensor* input = &session->tensors[lwm_read_u32(node + 8u)]; + const lw_runtime_tensor* weight = &session->tensors[lwm_read_u32(node + 12u)]; + const lw_runtime_tensor* output = &session->tensors[lwm_read_u32(node + 40u)]; + const uint8_t* params = model->bytes + (size_t)lwm_read_u64(node + 56u); + uint32_t groups = lwm_read_u32(params + 4u); + uint32_t kh = (uint32_t)lwm_read_i32(params + 8u); + uint32_t kw = (uint32_t)lwm_read_i32(params + 12u); + uint32_t sh = (uint32_t)lwm_read_i32(params + 16u); + uint32_t sw = (uint32_t)lwm_read_i32(params + 20u); + uint8_t node_layout = program->values[lwm_read_u32(node + 40u)].layout; + uint64_t count; + float* packed; + op->data.conv.input_channels = (uint32_t)input->dimensions[1]; + op->data.conv.input_offset = det_slot_offset(program, lwm_read_u32(node + 8u), node_layout); + op->data.conv.output_offset = program->values[lwm_read_u32(node + 40u)].offset; + op->data.conv.output_channels = (uint32_t)output->dimensions[1]; + op->data.conv.input_height = (uint32_t)input->dimensions[2]; + op->data.conv.input_width = (uint32_t)input->dimensions[3]; + op->data.conv.output_height = (uint32_t)output->dimensions[2]; + op->data.conv.output_width = (uint32_t)output->dimensions[3]; + op->data.conv.kernel_h = kh; op->data.conv.kernel_w = kw; + op->data.conv.weight_h = (uint32_t)weight->dimensions[2]; + op->data.conv.weight_w = (uint32_t)weight->dimensions[3]; + op->data.conv.stride_h = sh; op->data.conv.stride_w = sw; + op->data.conv.pad_top = (uint32_t)lwm_read_i32(params + 32u); + op->data.conv.pad_left = (uint32_t)lwm_read_i32(params + 36u); + op->data.conv.pad_bottom = (uint32_t)lwm_read_i32(params + 40u); + op->data.conv.pad_right = (uint32_t)lwm_read_i32(params + 44u); + op->data.conv.groups = groups; + if (node_layout == LW_X64_DET_LAYOUT_NCHW) { + const uint32_t weight_index = lwm_read_u32(node + 12u); + const float* weights = constant_f32(model, weight_index); + if (weights == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "NCHW Conv weights are not constant"); + return LW_STATUS_INVALID_ARGUMENT; + } + op->data.conv.original_weights = weights; + if (lwm_read_u16(node + 2u) >= 3u) { + op->data.conv.bias = constant_f32(model, lwm_read_u32(node + 16u)); + } + op->kind = LW_X64_DET_OP_CONV_NCHW; + op->data.conv.nchw_conv_kernel = LW_X64_DET_NCHW_CONV_SCALAR; + if (groups == 1u && kh == 1u && kw == 1u && sh == 1u && sw == 1u) { + if (!lw_packed_conv1x1_weight_count(op->data.conv.input_channels, + op->data.conv.output_channels, &count)) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "NCHW pointwise shape is unsupported"); + return LW_STATUS_UNSUPPORTED; + } + packed = (float*)alloc_constant(program, count * sizeof(float)); + if (packed == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "NCHW pointwise pack allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + lw_pack_conv1x1_weights_f32(weights, op->data.conv.input_channels, + op->data.conv.output_channels, packed); + op->data.conv.kernel_kind = LW_X64_DET_CONV_POINTWISE; + op->data.conv.packed_weights = packed; + op->data.conv.nchw_conv_kernel = LW_X64_DET_NCHW_CONV_POINTWISE_PACKED; + op->data.conv.nchw_pointwise_kernel = + choose_nchw_pointwise_kernel(op->data.conv.input_height, + op->data.conv.input_channels, + op->data.conv.output_channels, + op->data.conv.input_height * + op->data.conv.input_width); + return LW_STATUS_OK; + } + if (groups == 1u && kh == 3u && kw == 3u && sh == 2u && sw == 2u && + op->data.conv.pad_top == 1u && op->data.conv.pad_left == 1u && + op->data.conv.pad_bottom == 1u && op->data.conv.pad_right == 1u) { + op->data.conv.kernel_kind = LW_X64_DET_CONV_DENSE; + op->data.conv.nchw_conv_kernel = LW_X64_DET_NCHW_CONV_3X3S2; + return LW_STATUS_OK; + } + if (groups == 1u && kh == 3u && kw == 3u && sh == 1u && sw == 1u && + op->data.conv.pad_top == 1u && op->data.conv.pad_left == 1u && + op->data.conv.pad_bottom == 1u && op->data.conv.pad_right == 1u) { + op->data.conv.kernel_kind = LW_X64_DET_CONV_DENSE; + op->data.conv.nchw_conv_kernel = LW_X64_DET_NCHW_CONV_3X3_UNIT; + return LW_STATUS_OK; + } + if (groups == 1u && kh == 2u && kw == 2u && sh == 1u && sw == 1u && + op->data.conv.pad_top == 0u && op->data.conv.pad_left == 0u && + op->data.conv.pad_bottom == 1u && op->data.conv.pad_right == 1u) { + op->data.conv.kernel_kind = LW_X64_DET_CONV_DENSE; + op->data.conv.nchw_conv_kernel = LW_X64_DET_NCHW_CONV_2X2_PADEND1; + return LW_STATUS_OK; + } + if (det_conv_is_depthwise(input, weight, groups) && + kh == 3u && kw == 3u && + op->data.conv.pad_top == 1u && op->data.conv.pad_left == 1u && + op->data.conv.pad_bottom == 1u && op->data.conv.pad_right == 1u && + ((sh == 1u && sw == 1u) || (sh == 2u && sw == 1u))) { + op->data.conv.kernel_kind = LW_X64_DET_CONV_DEPTHWISE; + op->data.conv.nchw_conv_kernel = sh == 2u + ? LW_X64_DET_NCHW_CONV_DEPTHWISE3X3_S2X1 + : LW_X64_DET_NCHW_CONV_DEPTHWISE3X3; + return LW_STATUS_OK; + } + if (det_conv_is_depthwise(input, weight, groups) && + kh == 5u && kw == 5u && sh == 1u && sw == 1u && + op->data.conv.pad_top == 2u && op->data.conv.pad_left == 2u && + op->data.conv.pad_bottom == 2u && op->data.conv.pad_right == 2u) { + op->data.conv.kernel_kind = LW_X64_DET_CONV_DEPTHWISE; + op->data.conv.nchw_conv_kernel = LW_X64_DET_NCHW_CONV_DEPTHWISE5X5; + return LW_STATUS_OK; + } + /* Generic scalar fallback keeps every remaining shape correct. */ + op->data.conv.kernel_kind = det_conv_is_depthwise(input, weight, groups) + ? LW_X64_DET_CONV_DEPTHWISE : LW_X64_DET_CONV_DENSE; + return LW_STATUS_OK; + } + if (groups == 1u && kh == 1u && kw == 1u && sh == 1u && sw == 1u && + lw_nhwc_dense_packed_weight_count(op->data.conv.input_channels, + op->data.conv.output_channels, 1u, 1u, &count)) { + op->kind = LW_X64_DET_OP_POINTWISE; + op->data.conv.kernel_kind = LW_X64_DET_CONV_POINTWISE; + op->data.conv.pointwise_kernel = choose_pointwise_kernel( + op->data.conv.input_height * op->data.conv.input_width, + op->data.conv.output_channels); + } else if (groups == op->data.conv.input_channels && + groups == op->data.conv.output_channels && weight->dimensions[1] == 1u && + lw_nhwc_depthwise_packed_weight_count(op->data.conv.input_channels, kh, kw, + &count)) { + op->kind = LW_X64_DET_OP_DEPTHWISE; + op->data.conv.kernel_kind = LW_X64_DET_CONV_DEPTHWISE; + } else if (groups == 1u && + lw_nhwc_dense_packed_weight_count(op->data.conv.input_channels, + op->data.conv.output_channels, kh, kw, &count)) { + op->kind = LW_X64_DET_OP_DENSE; + op->data.conv.kernel_kind = LW_X64_DET_CONV_DENSE; + } else { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "Tiny DET Conv shape is not NHWC-lowerable"); + return LW_STATUS_UNSUPPORTED; + } + packed = (float*)alloc_constant(program, count * sizeof(float)); + if (packed == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "Conv packed weight allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + if (op->data.conv.kernel_kind == LW_X64_DET_CONV_DEPTHWISE) { + lw_pack_nhwc_depthwise_f32(constant_f32(model, lwm_read_u32(node + 12u)), + op->data.conv.input_channels, kh, kw, packed); + } else { + lw_pack_nhwc_dense_f32(constant_f32(model, lwm_read_u32(node + 12u)), + op->data.conv.input_channels, + op->data.conv.output_channels, kh, kw, packed); + } + op->data.conv.packed_weights = packed; + op->data.conv.original_weights = constant_f32(model, lwm_read_u32(node + 12u)); + if (lwm_read_u16(node + 2u) >= 3u) { + op->data.conv.bias = constant_f32(model, lwm_read_u32(node + 16u)); + } + op->data.conv.scratch_bytes = 0u; + if (op->kind == LW_X64_DET_OP_DENSE) { + lw_nhwc_dense_desc desc; + op->data.conv.dense_kc = LW_NHWC_DENSE_KC; + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; desc.input_channels = op->data.conv.input_channels; + desc.input_height = op->data.conv.input_height; desc.input_width = op->data.conv.input_width; + desc.output_channels = op->data.conv.output_channels; + desc.output_height = op->data.conv.output_height; desc.output_width = op->data.conv.output_width; + desc.kernel_h = kh; desc.kernel_w = kw; + desc.stride_h = sh; desc.stride_w = sw; + desc.pad_top = op->data.conv.pad_top; desc.pad_left = op->data.conv.pad_left; + desc.pad_bottom = op->data.conv.pad_bottom; desc.pad_right = op->data.conv.pad_right; + desc.dense_kc = op->data.conv.dense_kc; + if (!lw_nhwc_dense_scratch_bytes(&desc, &op->data.conv.scratch_bytes)) { + op->data.conv.scalar_fallback = 1u; + op->data.conv.scratch_bytes = 0u; + } + if (op->data.conv.scratch_bytes > program->scratch_bytes) { + program->scratch_bytes = op->data.conv.scratch_bytes; + } + } + return LW_STATUS_OK; +} + +static uint8_t node_eff_layout(const lw_x64_det_program* program, uint32_t node_index) { + return node_index < program->model->info.node_count + ? program->effective_node_layout[node_index] : LW_X64_DET_LAYOUT_NCHW; +} + +/* Offsets for one physical op input. The primary slot holds the producer's + * layout; when the consumer runs in the other layout, a LAYOUT_CONVERT op is + * materialized (once per value) and the consumer reads the alternate slot. */ +static uint64_t det_slot_offset(lw_x64_det_program* program, uint32_t tensor_index, + uint8_t consumer_layout) { + const lw_x64_det_value* value = &program->values[tensor_index]; + if (value->layout == consumer_layout || value->alt_producer < 0) return value->offset; + return value->alt_offset; +} + +static lw_status compile_node(const lw_model* model, const lw_session* session, + lw_x64_det_program* program, uint32_t node_index, + lw_x64_det_op* op, lw_error* error) { + const uint8_t* node = node_bytes(model, node_index); + const uint16_t semantic = lwm_read_u16(node); + uint32_t input_index = lwm_read_u32(node + 8u); + uint32_t output_index = lwm_read_u32(node + 40u); + const lw_runtime_tensor* input = &session->tensors[input_index]; + const lw_runtime_tensor* output = &session->tensors[output_index]; + const lw_x64_det_value* out_value = &program->values[output_index]; + const uint8_t* params = lwm_read_u64(node + 56u) == 0u + ? NULL : model->bytes + (size_t)lwm_read_u64(node + 56u); + uint64_t elements = value_elements(out_value); + uint8_t layout = out_value->layout; + op->semantic_begin = node_index; + op->semantic_count = 1u; + switch (semantic) { + case LW_OP_CONV: return compile_conv(model, session, program, node, op, error); + case LW_OP_RELU: op->kind = LW_X64_DET_OP_RELU; break; + case LW_OP_ERF: op->kind = LW_X64_DET_OP_ERF; break; + case LW_OP_HARD_SIGMOID: + op->kind = LW_X64_DET_OP_HARD_SIGMOID; + op->data.unary.alpha = read_f32(params + 4u); + op->data.unary.beta = read_f32(params + 8u); + break; + case LW_OP_SIGMOID: op->kind = LW_X64_DET_OP_SIGMOID_NCHW; break; + case LW_OP_BATCH_NORMALIZATION: { + uint32_t c = (uint32_t)input->dimensions[1]; + float* mul = (float*)alloc_constant(program, (uint64_t)c * sizeof(float)); + float* add = (float*)alloc_constant(program, (uint64_t)c * sizeof(float)); + uint32_t j; + if (mul == NULL || add == NULL) return LW_STATUS_OUT_OF_MEMORY; + for (j = 0u; j < c; ++j) { + float m = constant_f32(model, lwm_read_u32(node + 12u))[j]; + float b = constant_f32(model, lwm_read_u32(node + 16u))[j]; + float mean = constant_f32(model, lwm_read_u32(node + 20u))[j]; + float var = constant_f32(model, lwm_read_u32(node + 24u))[j]; + float scale = m / sqrtf(var + read_f32(params + 4u)); + mul[j] = scale; + add[j] = b - mean * scale; + } + op->kind = LW_X64_DET_OP_AFFINE; + op->data.affine.input_offset = det_slot_offset(program, input_index, layout); + op->data.affine.output_offset = out_value->offset; + op->data.affine.mul = mul; op->data.affine.add = add; + op->data.affine.scale = constant_f32(model, lwm_read_u32(node + 12u)); + op->data.affine.bias = constant_f32(model, lwm_read_u32(node + 16u)); + op->data.affine.mean = constant_f32(model, lwm_read_u32(node + 20u)); + op->data.affine.variance = constant_f32(model, lwm_read_u32(node + 24u)); + op->data.affine.epsilon = read_f32(params + 4u); + op->data.affine.channel_major = (uint8_t)(layout == LW_X64_DET_LAYOUT_NCHW || input->rank != 4u); + op->data.affine.pixels = (uint32_t)(value_elements(&program->values[input_index]) / c); + op->data.affine.channels = c; + break; + } + case LW_OP_ADD: case LW_OP_MUL: case LW_OP_DIV: case LW_OP_SUB: case LW_OP_POW: + op->kind = semantic == LW_OP_ADD ? LW_X64_DET_OP_ADD + : semantic == LW_OP_MUL ? LW_X64_DET_OP_MUL + : semantic == LW_OP_DIV ? LW_X64_DET_OP_DIV + : LW_X64_DET_OP_GENERIC_UNSUPPORTED; + op->data.binary.operation = semantic; + op->data.binary.element_count = elements; + op->data.binary.layout = layout; + op->data.binary.output_offset = out_value->offset; + op->data.binary.left_offset = det_slot_offset(program, input_index, layout); + op->data.binary.left_constant = program->values[input_index].constant_data; + op->data.binary.right_offset = det_slot_offset(program, lwm_read_u32(node + 12u), layout); + op->data.binary.right_constant = program->values[lwm_read_u32(node + 12u)].constant_data; + op->data.binary.broadcast_kind = + program->values[lwm_read_u32(node + 12u)].bytes == sizeof(float) + ? LW_X64_DET_BROADCAST_RIGHT_SCALAR + : program->values[lwm_read_u32(node + 12u)].bytes == out_value->bytes + ? LW_X64_DET_BROADCAST_SAME + : (program->values[lwm_read_u32(node + 12u)].bytes == + (uint64_t)out_value->dimensions[3] * sizeof(float)) + ? LW_X64_DET_BROADCAST_RIGHT_CHANNEL + : ((program->values[input_index].bytes == + (uint64_t)out_value->dimensions[3] * sizeof(float)) + ? LW_X64_DET_BROADCAST_LEFT_CHANNEL + : LW_X64_DET_BROADCAST_GENERAL); + memcpy(op->data.binary.dimensions, out_value->dimensions, + sizeof(op->data.binary.dimensions)); + op->data.binary.channels = out_value->rank > 0u + ? (uint32_t)out_value->dimensions[out_value->rank - 1u] : 1u; + op->data.binary.pixels = op->data.binary.channels != 0u + ? (uint32_t)(elements / op->data.binary.channels) : 0u; + if (layout == LW_X64_DET_LAYOUT_NCHW && out_value->rank == 4u) { + op->data.binary.channels = (uint32_t)out_value->dimensions[1]; + op->data.binary.pixels = op->data.binary.channels != 0u + ? (uint32_t)(elements / op->data.binary.channels) : 0u; + if (program->values[lwm_read_u32(node + 12u)].bytes == + (uint64_t)op->data.binary.channels * sizeof(float)) { + op->data.binary.broadcast_kind = LW_X64_DET_BROADCAST_RIGHT_CHANNEL; + } else if (program->values[input_index].bytes == + (uint64_t)op->data.binary.channels * sizeof(float)) { + op->data.binary.broadcast_kind = LW_X64_DET_BROADCAST_LEFT_CHANNEL; + } + } + break; + case LW_OP_REDUCE_MEAN: + op->kind = LW_X64_DET_OP_REDUCE_MEAN; + op->data.reduce.input_offset = det_slot_offset(program, input_index, layout); + op->data.reduce.output_offset = out_value->offset; + op->data.reduce.batch = (uint32_t)input->dimensions[0]; + op->data.reduce.height = (uint32_t)input->dimensions[2]; + op->data.reduce.width = (uint32_t)input->dimensions[3]; + op->data.reduce.channels = (uint32_t)input->dimensions[1]; + op->data.reduce.layout = layout; + break; + case LW_OP_AVERAGE_POOL: case LW_OP_MAX_POOL: + op->kind = semantic == LW_OP_MAX_POOL ? LW_X64_DET_OP_MAX_POOL : LW_X64_DET_OP_AVG_POOL; + op->data.pool.input_offset = det_slot_offset(program, input_index, layout); + op->data.pool.output_offset = out_value->offset; + op->data.pool.input_dimensions[0] = input->dimensions[0]; + op->data.pool.input_dimensions[1] = layout == LW_X64_DET_LAYOUT_NCHW ? input->dimensions[1] : input->dimensions[2]; + op->data.pool.input_dimensions[2] = layout == LW_X64_DET_LAYOUT_NCHW ? input->dimensions[2] : input->dimensions[3]; + op->data.pool.input_dimensions[3] = layout == LW_X64_DET_LAYOUT_NCHW ? input->dimensions[3] : input->dimensions[1]; + op->data.pool.output_dimensions[0] = output->dimensions[0]; + op->data.pool.output_dimensions[1] = layout == LW_X64_DET_LAYOUT_NCHW ? output->dimensions[1] : output->dimensions[2]; + op->data.pool.output_dimensions[2] = layout == LW_X64_DET_LAYOUT_NCHW ? output->dimensions[2] : output->dimensions[3]; + op->data.pool.output_dimensions[3] = layout == LW_X64_DET_LAYOUT_NCHW ? output->dimensions[3] : output->dimensions[1]; + op->data.pool.kernel[0] = lwm_read_i32(params + 8u); + op->data.pool.kernel[1] = lwm_read_i32(params + 12u); + op->data.pool.strides[0] = lwm_read_i32(params + 16u); + op->data.pool.strides[1] = lwm_read_i32(params + 20u); + op->data.pool.pads[0] = lwm_read_i32(params + 24u); + op->data.pool.pads[1] = lwm_read_i32(params + 28u); + op->data.pool.pads[2] = lwm_read_i32(params + 32u); + op->data.pool.pads[3] = lwm_read_i32(params + 36u); + op->data.pool.count_include_pad = (uint8_t)lwm_read_u32(params + 44u); + op->data.pool.is_max = (uint8_t)(semantic == LW_OP_MAX_POOL); + op->data.pool.layout = layout; + break; + case LW_OP_CONV_TRANSPOSE: { + /* Only the planner-capable 2x2/stride-2/no-pad shape (the + * probability-map head) is lowerable. */ + if (lwm_read_u32(params + 4u) != 1u || lwm_read_i32(params + 8u) != 2 || + lwm_read_i32(params + 12u) != 2 || lwm_read_i32(params + 16u) != 2 || + lwm_read_i32(params + 20u) != 2 || lwm_read_i32(params + 24u) != 1 || + lwm_read_i32(params + 28u) != 1 || lwm_read_i32(params + 32u) != 0 || + lwm_read_i32(params + 36u) != 0 || lwm_read_i32(params + 40u) != 0 || + lwm_read_i32(params + 44u) != 0) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "DET ConvTranspose shape is not lowerable"); + return LW_STATUS_UNSUPPORTED; + } + op->data.conv_transpose.input_offset = det_slot_offset(program, input_index, layout); + op->data.conv_transpose.output_offset = out_value->offset; + op->data.conv_transpose.weights = constant_f32(model, lwm_read_u32(node + 12u)); + op->data.conv_transpose.bias = lwm_read_u16(node + 2u) >= 3u + ? constant_f32(model, lwm_read_u32(node + 16u)) : NULL; + op->data.conv_transpose.input_dimensions[0] = input->dimensions[0]; + op->data.conv_transpose.input_dimensions[1] = input->dimensions[1]; + op->data.conv_transpose.input_dimensions[2] = input->dimensions[2]; + op->data.conv_transpose.input_dimensions[3] = input->dimensions[3]; + op->data.conv_transpose.output_dimensions[0] = output->dimensions[0]; + op->data.conv_transpose.output_dimensions[1] = output->dimensions[1]; + op->data.conv_transpose.output_dimensions[2] = output->dimensions[2]; + op->data.conv_transpose.output_dimensions[3] = output->dimensions[3]; + op->data.conv_transpose.layout = layout; + if (layout == LW_X64_DET_LAYOUT_NHWC) { + uint32_t oc = (uint32_t)output->dimensions[1]; + if (oc != 1u && (oc & 7u) != 0u) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "DET ConvTranspose output channels are not NHWC-lowerable"); + return LW_STATUS_UNSUPPORTED; + } + if (oc != 1u) { + uint64_t count = 0u; + float* packed; + if (!lw_nhwc_convtranspose_packed_weight_count( + op->data.conv_transpose.input_dimensions[1], oc, &count)) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "DET ConvTranspose pack shape is unsupported"); + return LW_STATUS_UNSUPPORTED; + } + packed = (float*)alloc_constant(program, count * sizeof(float)); + if (packed == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, + "DET ConvTranspose pack allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + lw_pack_nhwc_convtranspose2x2_f32( + op->data.conv_transpose.weights, + (uint32_t)op->data.conv_transpose.input_dimensions[1], oc, packed); + op->data.conv_transpose.packed_weights = packed; + op->kind = LW_X64_DET_OP_CONV_TRANSPOSE_NHWC; + } else { + op->kind = LW_X64_DET_OP_CONV_TRANSPOSE_NHWC_C1; + } + } else { + op->kind = LW_X64_DET_OP_CONV_TRANSPOSE_NCHW; + } + break; + } + case LW_OP_RESIZE: { + op->kind = layout == LW_X64_DET_LAYOUT_NHWC + ? LW_X64_DET_OP_RESIZE_NHWC : LW_X64_DET_OP_RESIZE_NCHW; + op->data.resize.input_offset = det_slot_offset(program, input_index, layout); + op->data.resize.output_offset = out_value->offset; + op->data.resize.rank = input->rank > 4u ? 4u : input->rank; + op->data.resize.input_dimensions[0] = input->dimensions[0]; + op->data.resize.input_dimensions[1] = input->dimensions[1]; + op->data.resize.input_dimensions[2] = input->dimensions[2]; + op->data.resize.input_dimensions[3] = input->dimensions[3]; + op->data.resize.output_dimensions[0] = output->dimensions[0]; + op->data.resize.output_dimensions[1] = output->dimensions[1]; + op->data.resize.output_dimensions[2] = output->dimensions[2]; + op->data.resize.output_dimensions[3] = output->dimensions[3]; + op->data.resize.scales[0] = read_f32(params + 4u); + op->data.resize.scales[1] = read_f32(params + 8u); + op->data.resize.scales[2] = read_f32(params + 12u); + op->data.resize.scales[3] = read_f32(params + 16u); + op->data.resize.layout = layout; + break; + } + case LW_OP_CONCAT: { + uint16_t input_count = lwm_read_u16(node + 2u); + uint16_t slot; + if (input_count > LWM_V0_MAX_NODE_INPUTS) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "DET Concat has too many inputs"); + return LW_STATUS_UNSUPPORTED; + } + op->kind = layout == LW_X64_DET_LAYOUT_NHWC + ? LW_X64_DET_OP_CONCAT_NHWC : LW_X64_DET_OP_CONCAT_NCHW; + op->data.concat.input_count = input_count; + op->data.concat.output_offset = out_value->offset; + op->data.concat.axis = lwm_read_i32(params + 4u); + op->data.concat.output_rank = output->rank > 4u ? 4u : output->rank; + op->data.concat.layout = layout; + for (slot = 0u; slot < input_count; ++slot) { + const lw_runtime_tensor* concat_input = + &session->tensors[lwm_read_u32(node + 8u + (size_t)slot * 4u)]; + op->data.concat.input_offsets[slot] = + det_slot_offset(program, lwm_read_u32(node + 8u + (size_t)slot * 4u), layout); + op->data.concat.input_ranks[slot] = concat_input->rank > 4u ? 4u : concat_input->rank; + memcpy(op->data.concat.input_dimensions[slot], concat_input->dimensions, + sizeof(op->data.concat.input_dimensions[slot])); + } + memcpy(op->data.concat.output_dimensions, output->dimensions, + sizeof(op->data.concat.output_dimensions)); + break; + } + default: + op->kind = LW_X64_DET_OP_GENERIC_UNSUPPORTED; + break; + } + if (op->kind == LW_X64_DET_OP_RELU || op->kind == LW_X64_DET_OP_ERF || + op->kind == LW_X64_DET_OP_HARD_SIGMOID || op->kind == LW_X64_DET_OP_SIGMOID_NCHW) { + op->data.unary.input_offset = det_slot_offset(program, input_index, layout); + op->data.unary.output_offset = out_value->offset; + op->data.unary.element_count = elements; + op->data.unary.rank = out_value->rank; + op->data.unary.layout = layout; + memcpy(op->data.unary.dimensions, out_value->dimensions, + sizeof(op->data.unary.dimensions)); + } + return LW_STATUS_OK; +} + +static uint32_t resolve_alias_root(const lw_x64_det_program* program, uint32_t index) { + uint32_t guard = 0u; + while (program->values[index].alias != 0u && guard++ < program->value_count) { + index = program->values[index].alias_of; + } + return index; +} + +/* Lowerability: every planner-NHWC node except the island-exit set. Sigmoid + * stays NCHW-scalar forever (canonical parity and neutral tensors make it + * conversion-free); SQRT is not in the lowering switch. The planner's shape + * rules gate the rest. */ +static int det_node_nhwc_lowerable(uint16_t operation) { + switch (operation) { + case LW_OP_CONV: + case LW_OP_ADD: + case LW_OP_MUL: + case LW_OP_DIV: + case LW_OP_SUB: + case LW_OP_POW: + case LW_OP_RELU: + case LW_OP_ERF: + case LW_OP_HARD_SIGMOID: + case LW_OP_BATCH_NORMALIZATION: + case LW_OP_REDUCE_MEAN: + case LW_OP_AVERAGE_POOL: + case LW_OP_MAX_POOL: + case LW_OP_RESIZE: + case LW_OP_CONV_TRANSPOSE: + case LW_OP_CONCAT: + return 1; + default: + return 0; + } +} + +/* Cross the planner's NHWC classification with the v1 lowerability set and + * force matched GELU patterns to a uniform layout. */ +static lw_status plan_effective_layouts(const lw_model* model, const lw_session* session, + const lw_layout_plan* plan, + lw_x64_det_program* program, lw_error* error) { + uint32_t node_index; + program->effective_node_layout = (uint8_t*)calloc(model->info.node_count, sizeof(uint8_t)); + if (program->effective_node_layout == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "DET effective layout allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (node_index = 0u; node_index < model->info.node_count; ++node_index) { + const uint8_t* node = node_bytes(model, node_index); + uint16_t operation = lwm_read_u16(node); + if (plan->node_layout[node_index] == LW_FAST_LAYOUT_NHWC && + det_node_nhwc_lowerable(operation)) { + program->effective_node_layout[node_index] = LW_X64_DET_LAYOUT_NHWC; + ++program->nhwc_effective_nodes; + } else { + if (plan->node_layout[node_index] == LW_FAST_LAYOUT_NHWC && + operation != LW_OP_SIGMOID && operation != LW_OP_SQRT) { + /* Planner/backend divergence: the planner classified an op + * NHWC that this backend cannot lower in any version. */ + char message[128]; + (void)snprintf(message, sizeof(message), + "DET node %" PRIu32 " (op %u) is planner-NHWC but not lowerable", + (unsigned)node_index, (unsigned)operation); + lw_set_error(error, LW_STATUS_UNSUPPORTED, message); + return LW_STATUS_UNSUPPORTED; + } + program->effective_node_layout[node_index] = LW_X64_DET_LAYOUT_NCHW; + ++program->nchw_effective_nodes; + } + } + for (node_index = 0u; node_index + 5u <= model->info.node_count; ++node_index) { + lw_fused_gelu_match match; + if (lw_match_fused_gelu(session, node_index, &match)) { + uint8_t layout = program->effective_node_layout[node_index]; + uint32_t fused; + for (fused = node_index + 1u; fused < node_index + 5u; ++fused) { + program->effective_node_layout[fused] = layout; + } + node_index += 4u; + } + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +typedef struct det_arena_candidate { + uint32_t value_index; + uint64_t bytes; + int32_t first; + int32_t last; + uint64_t offset; + uint8_t is_alt; +} det_arena_candidate; + +static int det_candidate_before(const det_arena_candidate* a, const det_arena_candidate* b) { + if (a->bytes != b->bytes) return a->bytes > b->bytes; + if (a->first != b->first) return a->first < b->first; + if (a->value_index != b->value_index) return a->value_index < b->value_index; + return a->is_alt < b->is_alt; +} + +/* Copy of the proven REC greedy-by-size interval planner, extended with + * alternate-layout slots: values read across an effective-layout boundary get + * a second candidate whose lifetime spans its LAYOUT_CONVERT op to the last + * alternate-layout consumer. */ +static lw_status plan_arena_offsets(lw_x64_det_program* program, lw_error* error) { + det_arena_candidate* candidates; + uint32_t count = 0u; + uint32_t i; + uint64_t arena_bytes = 0u; + candidates = (det_arena_candidate*)calloc(program->value_count * 2u, sizeof(*candidates)); + if (candidates == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "DET arena plan allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (i = 0u; i < program->value_count; ++i) { + const lw_x64_det_value* value = &program->values[i]; + int32_t first; + int32_t last; + uint32_t a; + if (value->alias != 0u || value->constant_data != NULL) continue; + if ((int32_t)i == (int32_t)program->input_value) { + first = 0; + last = (int32_t)program->op_count; + } else if (value->producer >= 0) { + first = value->producer; + last = value->last_use > value->producer ? value->last_use : value->producer; + } else { + continue; + } + for (a = 0u; a < program->value_count; ++a) { + const lw_x64_det_value* alias = &program->values[a]; + if (alias->alias != 0u && resolve_alias_root(program, a) == i && + alias->last_use > last) { + last = alias->last_use; + } + } + candidates[count].value_index = i; + candidates[count].bytes = value->bytes; + candidates[count].first = first; + candidates[count].last = last; + ++count; + if (value->alt_producer >= 0) { + candidates[count].value_index = i; + candidates[count].bytes = value->bytes; + candidates[count].first = value->alt_producer; + candidates[count].last = value->alt_last_use > value->alt_producer + ? value->alt_last_use : value->alt_producer; + candidates[count].is_alt = 1u; + ++count; + } + } + for (i = 1u; i < count; ++i) { + det_arena_candidate current = candidates[i]; + uint32_t j = i; + while (j > 0u && det_candidate_before(¤t, &candidates[j - 1u])) { + candidates[j] = candidates[j - 1u]; + --j; + } + candidates[j] = current; + } + for (i = 0u; i < count; ++i) { + det_arena_candidate* current = &candidates[i]; + uint64_t offset = 0u; + int collided = 1; + while (collided) { + uint32_t p; + collided = 0; + for (p = 0u; p < i; ++p) { + const det_arena_candidate* placed = &candidates[p]; + if (placed->first > current->last || current->first > placed->last) continue; + if (offset < placed->offset + placed->bytes && + placed->offset < offset + current->bytes) { + offset = lw_x64_rec_arena_align(placed->offset + placed->bytes, 64u); + if (offset == UINT64_MAX) { + free(candidates); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, + "DET arena plan overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + collided = 1; + break; + } + } + } + offset = lw_x64_rec_arena_align(offset, 64u); + if (offset == UINT64_MAX || current->bytes > UINT64_MAX - offset) { + free(candidates); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "DET arena plan overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + current->offset = offset; + if (offset + current->bytes > arena_bytes) arena_bytes = offset + current->bytes; + if (current->is_alt) { + program->values[current->value_index].alt_offset = offset; + } else { + program->values[current->value_index].offset = offset; + } + } + for (i = 0u; i < program->value_count; ++i) { + if (program->values[i].alias != 0u) { + const lw_x64_det_value* root = &program->values[resolve_alias_root(program, i)]; + program->values[i].offset = root->offset; + program->values[i].alt_offset = root->alt_offset; + } + } + program->arena_bytes = arena_bytes; + free(candidates); + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +static void reset_lowering(lw_x64_det_program* program, uint32_t limit) { + uint32_t i; + for (i = 0u; i < program->packed_constant_count; ++i) free(program->constants[i].data); + free(program->constants); + program->constants = NULL; + program->packed_constant_count = 0u; + memset(program->ops, 0, (size_t)limit * sizeof(*program->ops)); + program->op_count = 0u; + program->semantic_consumed = 0u; + program->semantic_elided = 0u; + program->scratch_bytes = 0u; + program->unsupported_nodes = 0u; + for (i = 0u; i < program->value_count; ++i) { + program->values[i].producer = -1; + program->values[i].last_use = -1; + program->values[i].alt_producer = -1; + program->values[i].alt_last_use = -1; + } +} + +static lw_x64_det_compile_result lower_ops(const lw_model* model, const lw_session* session, + lw_x64_det_program* program, uint32_t limit, + lw_error* error); + +static uint32_t det_tensor_consumer_count(const lw_model* model, uint32_t tensor_index) { + uint32_t count = 0u; + uint32_t node; + for (node = 0u; node < model->info.node_count; ++node) { + const uint8_t* current = node_bytes(model, node); + uint16_t input_count = lwm_read_u16(current + 2u); + uint16_t input_index; + for (input_index = 0u; input_index < input_count; ++input_index) { + if (lwm_read_u32(current + 8u + (size_t)input_index * sizeof(uint32_t)) == + tensor_index) { + ++count; + } + } + } + return count; +} + +/* Every consumer of the tensor must be one of the `span` nodes starting at + * `first`, and the tensor must not be a graph output. The GELU chain reads + * its input twice, so a plain single-consumer test is too strict. */ +static int det_tensor_consumed_within(const lw_model* model, uint32_t tensor, uint32_t first, + uint32_t span) { + uint32_t node; + uint32_t output; + int used = 0; + for (node = 0u; node < model->info.node_count; ++node) { + const uint8_t* current = node_bytes(model, node); + uint16_t input_count = lwm_read_u16(current + 2u); + uint16_t input_index; + for (input_index = 0u; input_index < input_count; ++input_index) { + if (lwm_read_u32(current + 8u + (size_t)input_index * sizeof(uint32_t)) == tensor) { + if (node < first || node >= first + span) { + return 0; + } + used = 1; + } + } + } + for (output = 0u; output < model->info.output_count; ++output) { + if (lwm_read_u32(model->bytes + (size_t)model->output_offset + + (size_t)output * sizeof(uint32_t)) == tensor) { + return 0; + } + } + return used; +} + +static int det_input_is_neutral(const lw_session* session, uint32_t tensor_index) { + return tensor_index < session->model->info.tensor_count && + lw_layout_tensor_is_neutral(session, tensor_index); +} + +/* Layout conversion only applies to rank-4 non-neutral tensors: neutral + * tensors are layout-identical and other ranks are layout-agnostic. */ +static int det_tensor_convertible(const lw_session* session, uint32_t tensor_index) { + return tensor_index < session->model->info.tensor_count && + session->tensors[tensor_index].rank == 4u && + !det_input_is_neutral(session, tensor_index); +} + +/* Materialize LAYOUT_CONVERT ops for every activation input of `node_index` + * whose storage layout differs from the node's effective layout. Neutral + * tensors ([N,1,H,W] or [N,C,1,1]) are layout-identical and skipped. */ +static lw_status materialize_input_converts(const lw_model* model, const lw_session* session, + lw_x64_det_program* program, uint32_t node_index, + uint32_t* physical, uint32_t limit, + lw_error* error) { + const uint8_t* node = node_bytes(model, node_index); + uint16_t input_count = lwm_read_u16(node + 2u); + uint8_t node_layout = node_eff_layout(program, node_index); + uint16_t input_slot; + for (input_slot = 0u; input_slot < input_count; ++input_slot) { + uint32_t tensor_index = lwm_read_u32(node + 8u + (size_t)input_slot * 4u); + lw_x64_det_value* value; + if (tensor_index >= program->value_count) continue; + value = &program->values[tensor_index]; + if (value->alias != 0u) { + value = &program->values[resolve_alias_root(program, tensor_index)]; + } + if (value->constant_data != NULL || value->layout == node_layout || + !det_tensor_convertible(session, tensor_index)) { + continue; + } + if (value->alt_producer >= 0) { + if (value->alt_layout != node_layout) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "DET tensor requires three layouts"); + return LW_STATUS_UNSUPPORTED; + } + continue; + } + if (*physical >= limit) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "DET physical op table overflow"); + return LW_STATUS_OUT_OF_BOUNDS; + } + { + const lw_runtime_tensor* tensor = &session->tensors[tensor_index]; + lw_x64_det_op* convert = push_op(program, physical, LW_X64_DET_OP_LAYOUT_CONVERT, + node_index, 0u); + convert->data.convert.input_offset = value->offset; + convert->data.convert.output_offset = value->alt_offset; + convert->data.convert.batch = (uint32_t)tensor->dimensions[0]; + convert->data.convert.channels = (uint32_t)tensor->dimensions[1]; + convert->data.convert.height = (uint32_t)tensor->dimensions[2]; + convert->data.convert.width = (uint32_t)tensor->dimensions[3]; + convert->data.convert.to_nhwc = (uint8_t)(node_layout == LW_X64_DET_LAYOUT_NHWC); + } + value->alt_producer = (int32_t)(*physical - 1u); + value->alt_layout = node_layout; + if (value->last_use < value->alt_producer) value->last_use = value->alt_producer; + } + return LW_STATUS_OK; +} + +/* Update input read lifetimes for one semantic node whose physical op sits at + * `physical`. Primary-slot reads extend last_use; alternate-slot reads extend + * alt_last_use. */ +static void update_input_uses(const lw_model* model, const lw_session* session, + lw_x64_det_program* program, uint32_t node_index, + int32_t physical) { + const uint8_t* node = node_bytes(model, node_index); + uint16_t input_count = lwm_read_u16(node + 2u); + uint8_t node_layout = node_eff_layout(program, node_index); + uint16_t input_slot; + for (input_slot = 0u; input_slot < input_count; ++input_slot) { + uint32_t tensor_index = lwm_read_u32(node + 8u + (size_t)input_slot * 4u); + lw_x64_det_value* value; + if (tensor_index >= program->value_count) continue; + value = &program->values[tensor_index]; + if (value->alias != 0u) { + value = &program->values[resolve_alias_root(program, tensor_index)]; + } + if (value->constant_data != NULL) continue; + if (value->layout != node_layout && value->alt_producer >= 0 && + det_tensor_convertible(session, tensor_index)) { + if (value->alt_last_use < physical) value->alt_last_use = physical; + } else if (value->last_use < physical) { + value->last_use = physical; + } + } +} + +static lw_x64_det_compile_result lower_ops(const lw_model* model, const lw_session* session, + lw_x64_det_program* program, uint32_t limit, + lw_error* error) { + uint32_t i; + uint32_t physical = 0u; + lw_status status = LW_STATUS_OK; + for (i = 0u; i < model->info.node_count; ++i) { + const uint8_t* node = node_bytes(model, i); + uint16_t semantic = lwm_read_u16(node); + { + lw_fused_gelu_match match; + if (i + 5u <= model->info.node_count && lw_match_fused_gelu(session, i, &match) && + fused_gelu_temporaries_private(model, i, &match) && physical < limit) { + uint8_t layout = node_eff_layout(program, i); + uint32_t fused_node; + lw_x64_det_op* op; + const lw_x64_det_value* output = &program->values[match.outputs[4]]; + for (fused_node = i; fused_node < i + 5u; ++fused_node) { + status = materialize_input_converts(model, session, program, fused_node, + &physical, limit, error); + if (status != LW_STATUS_OK) return status == LW_STATUS_OUT_OF_MEMORY + ? LW_X64_DET_COMPILE_OUT_OF_MEMORY : LW_X64_DET_COMPILE_UNSUPPORTED; + } + if (physical >= limit) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "DET physical op table overflow"); + return LW_X64_DET_COMPILE_INVALID_GRAPH; + } + op = &program->ops[physical]; + memset(op, 0, sizeof(*op)); + op->kind = LW_X64_DET_OP_GELU; + op->semantic_begin = i; + op->semantic_count = 5u; + op->data.unary.input_offset = + det_slot_offset(program, match.inputs[0][0], layout); + op->data.unary.output_offset = output->offset; + op->data.unary.element_count = output->bytes / sizeof(float); + op->data.unary.rank = output->rank; + op->data.unary.layout = layout; + memcpy(op->data.unary.dimensions, output->dimensions, + sizeof(op->data.unary.dimensions)); + program->values[match.outputs[4]].producer = (int32_t)physical; + for (fused_node = i; fused_node < i + 5u; ++fused_node) { + update_input_uses(model, session, program, fused_node, (int32_t)physical); + } + ++physical; + program->semantic_consumed += 5u; + i += 4u; + continue; + } + } + if (semantic == LW_OP_RESHAPE || semantic == LW_OP_SQUEEZE || + semantic == LW_OP_UNSQUEEZE) { + uint32_t in = lwm_read_u32(node + 8u); + uint32_t out = lwm_read_u32(node + 40u); + program->values[out].alias = 1u; + program->values[out].alias_of = in; + program->values[out].offset = program->values[in].offset; + ++program->semantic_elided; + program->values[out].producer = (int32_t)i; + continue; + } + status = materialize_input_converts(model, session, program, i, &physical, limit, error); + if (status != LW_STATUS_OK) { + return status == LW_STATUS_OUT_OF_MEMORY ? LW_X64_DET_COMPILE_OUT_OF_MEMORY + : LW_X64_DET_COMPILE_UNSUPPORTED; + } + if (physical >= limit) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "DET physical op table overflow"); + return LW_X64_DET_COMPILE_INVALID_GRAPH; + } + { + lw_x64_det_op* op = &program->ops[physical]; + memset(op, 0, sizeof(*op)); + status = compile_node(model, session, program, i, op, error); + if (status != LW_STATUS_OK || op->kind == LW_X64_DET_OP_GENERIC_UNSUPPORTED) { + ++program->unsupported_nodes; + if (status == LW_STATUS_OK || error == NULL || error->message[0] == 0) { + char message[128]; + (void)snprintf(message, sizeof(message), + "DET node %u (op %u) lowering failed (status %d, kind %u)", + (unsigned)i, (unsigned)semantic, (int)status, + (unsigned)op->kind); + lw_set_error(error, status != LW_STATUS_OK ? status : LW_STATUS_UNSUPPORTED, + message); + } + return LW_X64_DET_COMPILE_UNSUPPORTED; + } + program->semantic_consumed++; + program->values[lwm_read_u32(node + 40u)].producer = (int32_t)physical; + update_input_uses(model, session, program, i, (int32_t)physical); + ++physical; + /* Conv epilogue folding: greedily absorb a trailing channel-bias + * Add, a residual Add, and a final Relu or five-node exact-GELU + * chain into an NHWC pointwise/dense conv's fused epilogue. The + * kernels apply accumulator -> +post_bias -> +residual -> + * activation in exactly the original op order, so results stay + * bit-identical to the unfused sequence. Elided intermediate + * tensors never materialize (producer/last_use = -1). */ + if (semantic == LW_OP_CONV) { + lw_x64_det_op* conv_op = &program->ops[physical - 1u]; + uint32_t conv_output = lwm_read_u32(node + 40u); + if ((conv_op->kind == LW_X64_DET_OP_POINTWISE || + conv_op->kind == LW_X64_DET_OP_DENSE) && + conv_op->data.conv.activation == LW_NHWC_ACT_NONE && + program->values[conv_output].layout == LW_X64_DET_LAYOUT_NHWC) { + uint32_t current = conv_output; + uint64_t current_bytes = program->values[current].bytes; + uint32_t cursor = i + 1u; + uint32_t consumed = 0u; + uint32_t residual_tensor = UINT32_MAX; + uint32_t final_out = current; + const float* post_bias = NULL; + uint16_t activation = LW_NHWC_ACT_NONE; + uint32_t intermediates[3]; + uint32_t intermediate_count = 0u; + /* Channel-bias Add and residual Add, in either order, at + * most one of each. Both must be NHWC-effective so no + * layout convert would have sat between conv and Add. */ + while (cursor < model->info.node_count && + (post_bias == NULL || residual_tensor == UINT32_MAX)) { + const uint8_t* add = node_bytes(model, cursor); + uint32_t left; + uint32_t right; + uint32_t other; + uint32_t add_out; + uint32_t root; + lw_x64_det_value* other_value; + if (lwm_read_u16(add) != LW_OP_ADD || + lwm_read_u16(add + 2u) != 2u || + node_eff_layout(program, cursor) != LW_X64_DET_LAYOUT_NHWC || + det_tensor_consumer_count(model, current) != 1u) { + break; + } + left = lwm_read_u32(add + 8u); + right = lwm_read_u32(add + 12u); + other = left == current ? right + : right == current ? left : UINT32_MAX; + if (other == UINT32_MAX || other == current) break; + add_out = lwm_read_u32(add + 40u); + if (add_out == current || + program->values[add_out].bytes != current_bytes || + program->values[add_out].alias != 0u) { + break; + } + root = resolve_alias_root(program, other); + other_value = &program->values[root]; + if (other_value->constant_data != NULL) { + if (post_bias != NULL || + other_value->bytes != + (uint64_t)conv_op->data.conv.output_channels * + sizeof(float)) { + break; + } + post_bias = other_value->constant_data; + } else { + if (residual_tensor != UINT32_MAX || + other_value->bytes != current_bytes || + other_value->layout != LW_X64_DET_LAYOUT_NHWC) { + break; + } + residual_tensor = root; + } + intermediates[intermediate_count++] = current; + current = add_out; + final_out = add_out; + ++consumed; + ++cursor; + } + /* Relu or five-node exact-GELU chain after the adds. The + * GELU chain reads its input twice (Div and outer Mul), + * so require every consumer to live inside the chain. */ + if (cursor < model->info.node_count) { + const uint8_t* next = node_bytes(model, cursor); + if (lwm_read_u16(next) == LW_OP_RELU && + lwm_read_u32(next + 8u) == current && + det_tensor_consumer_count(model, current) == 1u) { + intermediates[intermediate_count++] = current; + activation = LW_NHWC_ACT_RELU; + final_out = lwm_read_u32(next + 40u); + ++consumed; + } else { + lw_fused_gelu_match match; + if (cursor + 5u <= model->info.node_count && + lw_match_fused_gelu(session, cursor, &match) && + match.inputs[0][0] == current && + fused_gelu_temporaries_private(model, cursor, &match) && + det_tensor_consumed_within(model, current, cursor, 5u)) { + intermediates[intermediate_count++] = current; + activation = LW_NHWC_ACT_GELU; + final_out = match.outputs[4]; + consumed += 5u; + } + } + } + if (consumed > 0u) { + uint32_t k; + conv_op->data.conv.post_bias = post_bias; + if (residual_tensor != UINT32_MAX) { + conv_op->data.conv.has_residual = 1u; + conv_op->data.conv.residual_offset = + det_slot_offset(program, residual_tensor, + LW_X64_DET_LAYOUT_NHWC); + if (program->values[residual_tensor].last_use < + (int32_t)(physical - 1u)) { + program->values[residual_tensor].last_use = + (int32_t)(physical - 1u); + } + } + conv_op->data.conv.activation = activation; + conv_op->data.conv.output_offset = + program->values[final_out].offset; + conv_op->semantic_count = (uint16_t)(1u + consumed); + for (k = 0u; k < intermediate_count; ++k) { + program->values[intermediates[k]].producer = -1; + program->values[intermediates[k]].last_use = -1; + } + program->values[final_out].producer = (int32_t)(physical - 1u); + program->semantic_elided += consumed; + i += consumed; + continue; + } + } + } + } + } + /* Graph-output tensors are read by the caller in NCHW; if a producer left + * one in NHWC (possible once Phase 2 makes the tail NHWC), materialize a + * trailing convert. Neutral outputs are layout-identical and skipped. */ + { + uint32_t t; + for (t = 0u; t < program->value_count; ++t) { + lw_x64_det_value* value = &program->values[t]; + int32_t birth; + if ((session->tensors[t].flags & LWM_V0_TENSOR_FLAG_OUTPUT) == 0u || + det_input_is_neutral(session, t)) { + continue; + } + birth = session->tensors[t].birth_node; + if (value->layout != LW_X64_DET_LAYOUT_NHWC || birth < 0 || + node_eff_layout(program, (uint32_t)birth) != LW_X64_DET_LAYOUT_NHWC) { + continue; + } + if (physical >= limit) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "DET physical op table overflow"); + return LW_X64_DET_COMPILE_INVALID_GRAPH; + } + { + const lw_runtime_tensor* tensor = &session->tensors[t]; + lw_x64_det_op* convert = push_op(program, &physical, + LW_X64_DET_OP_LAYOUT_CONVERT, + (uint32_t)birth, 0u); + convert->data.convert.input_offset = value->offset; + convert->data.convert.output_offset = value->alt_offset; + convert->data.convert.batch = (uint32_t)tensor->dimensions[0]; + convert->data.convert.channels = (uint32_t)tensor->dimensions[1]; + convert->data.convert.height = (uint32_t)tensor->dimensions[2]; + convert->data.convert.width = (uint32_t)tensor->dimensions[3]; + convert->data.convert.to_nhwc = 0u; + } + value->alt_producer = (int32_t)(physical - 1u); + value->alt_layout = LW_X64_DET_LAYOUT_NCHW; + if (value->last_use < value->alt_producer) value->last_use = value->alt_producer; + } + } + program->op_count = physical; + return LW_X64_DET_COMPILE_OK; +} + +lw_x64_det_compile_result lw_x64_det_backend_compile_ex( + const lw_model* model, uint32_t height, uint32_t width, + lw_x64_det_compile_strategy strategy, lw_x64_det_program** out_program, lw_error* error) { + lw_session* session = NULL; + lw_x64_det_program* program = NULL; + lw_model_info info; + lw_status status; + uint32_t i; + uint32_t capacity; + lw_x64_det_compile_result lowered; + if (out_program == NULL || model == NULL || height == 0u || width == 0u || + height > INT32_MAX || width > INT32_MAX) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "DET backend arguments are invalid"); + return LW_X64_DET_COMPILE_INVALID_GRAPH; + } + *out_program = NULL; + lw_model_info_init(&info); + status = lw_model_get_info(model, &info); + if (status != LW_STATUS_OK) return LW_X64_DET_COMPILE_INVALID_GRAPH; + status = make_shape_session(model, height, width, &session, error); + if (status != LW_STATUS_OK) { + return status == LW_STATUS_OUT_OF_MEMORY ? LW_X64_DET_COMPILE_OUT_OF_MEMORY + : LW_X64_DET_COMPILE_INVALID_GRAPH; + } + program = (lw_x64_det_program*)calloc(1u, sizeof(*program)); + if (program == NULL) { + lw_session_free(session); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "DET program allocation failed"); + return LW_X64_DET_COMPILE_OUT_OF_MEMORY; + } + program->model = model; + program->cpu = lw_get_cpu_capabilities(); + program->model_signature = info.content_checksum; + program->input_height = height; + program->input_width = width; + program->backend_layout = (uint8_t)strategy; + program->value_count = info.tensor_count; +#if defined(__EMSCRIPTEN__) && defined(__wasm_simd128__) + if (strategy != LW_X64_DET_COMPILE_NHWC) { + lw_session_free(session); + lw_x64_det_program_free(program); + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "WASM compiled DET supports NHWC layout only"); + return LW_X64_DET_COMPILE_UNSUPPORTED; + } +#else + if (!lw_simd_level_is_avx2(program->cpu.simd) || !program->cpu.has_avx2_fma) { + lw_session_free(session); + lw_x64_det_program_free(program); + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "standalone x64 DET backend requires AVX2 and FMA"); + return LW_X64_DET_COMPILE_UNSUPPORTED; + } +#endif + program->effective_node_layout = (uint8_t*)calloc(info.node_count, sizeof(uint8_t)); + if (program->effective_node_layout == NULL) { + lw_session_free(session); + lw_x64_det_program_free(program); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "DET effective layout allocation failed"); + return LW_X64_DET_COMPILE_OUT_OF_MEMORY; + } + if (strategy == LW_X64_DET_COMPILE_NHWC) { + lw_layout_planner_options options; + lw_layout_plan plan; + memset(&options, 0, sizeof(options)); + options.allow_direct_nhwc_graph_input = 1u; + memset(&plan, 0, sizeof(plan)); + status = lw_layout_plan_build(session, &options, &plan, error); + if (status != LW_STATUS_OK) { + lw_layout_plan_free(&plan); + lw_session_free(session); + lw_x64_det_program_free(program); + return LW_X64_DET_COMPILE_INVALID_GRAPH; + } + program->direct_nhwc = plan.graph_input_direct_nhwc; + status = plan_effective_layouts(model, session, &plan, program, error); + lw_layout_plan_free(&plan); + if (status != LW_STATUS_OK) { + lw_session_free(session); + lw_x64_det_program_free(program); + return status == LW_STATUS_OUT_OF_MEMORY ? LW_X64_DET_COMPILE_OUT_OF_MEMORY + : LW_X64_DET_COMPILE_UNSUPPORTED; + } + } else { + program->nchw_effective_nodes = info.node_count; + } + /* Physical ops: one per semantic node at most, plus one shared convert per + * value at most. */ + capacity = info.node_count + info.tensor_count + 16u; + program->ops = (lw_x64_det_op*)calloc(capacity, sizeof(*program->ops)); + program->values = (lw_x64_det_value*)calloc(program->value_count, + sizeof(*program->values)); + if (program->ops == NULL || program->values == NULL) { + lw_session_free(session); + lw_x64_det_program_free(program); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "DET program tables allocation failed"); + return LW_X64_DET_COMPILE_OUT_OF_MEMORY; + } + for (i = 0u; i < program->value_count; ++i) { + fill_value(session, i, &program->values[i], program); + if ((session->tensors[i].flags & LWM_V0_TENSOR_FLAG_INPUT) != 0u) { + program->input_value = i; + } + } + lowered = lower_ops(model, session, program, capacity, error); + if (lowered != LW_X64_DET_COMPILE_OK) { + lw_session_free(session); + lw_x64_det_program_free(program); + return lowered; + } + status = plan_arena_offsets(program, error); + if (status != LW_STATUS_OK) { + lw_session_free(session); + lw_x64_det_program_free(program); + return status == LW_STATUS_OUT_OF_MEMORY ? LW_X64_DET_COMPILE_OUT_OF_MEMORY + : LW_X64_DET_COMPILE_INVALID_GRAPH; + } + reset_lowering(program, capacity); + lowered = lower_ops(model, session, program, capacity, error); + if (lowered != LW_X64_DET_COMPILE_OK) { + lw_session_free(session); + lw_x64_det_program_free(program); + return lowered; + } + program->semantic_consumed += program->semantic_elided; + program->output_value = info.output_count + ? lwm_read_u32(model->bytes + (size_t)model->output_offset) : 0u; + for (i = 0u; i < program->op_count; ++i) { + if (program->ops[i].kind == LW_X64_DET_OP_LAYOUT_CONVERT) { + ++program->layout_conversions; + } + } + if (program->arena_bytes == 0u) { + lw_session_free(session); + lw_x64_det_program_free(program); + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "DET graph has no runtime arena"); + return LW_X64_DET_COMPILE_INVALID_GRAPH; + } + lw_session_free(session); + lw_set_error(error, LW_STATUS_OK, ""); + *out_program = program; + return LW_X64_DET_COMPILE_OK; +} + +void lw_x64_det_program_free(lw_x64_det_program* program) { + uint32_t i; + if (program == NULL) return; + for (i = 0u; i < program->packed_constant_count; ++i) free(program->constants[i].data); + free(program->constants); + free(program->ops); + free(program->values); + free(program->effective_node_layout); + free(program); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_det_backend_execute.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_det_backend_execute.c new file mode 100644 index 00000000..5649631e --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_det_backend_execute.c @@ -0,0 +1,1612 @@ +#include "x64_det_backend_internal.h" +#include "lwm_read.h" +#include "operator_internal.h" +#include "parallel_internal.h" +#include "../ppocr/profile_internal.h" +#include "rec_backend_kernels_internal.h" +#include "../simd/simd_kernels.h" + +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#include +static void* det_aligned_alloc(size_t alignment, size_t size) { return _aligned_malloc(size, alignment); } +static void det_aligned_free(void* p) { _aligned_free(p); } +#else +static void* det_aligned_alloc(size_t alignment, size_t size) { + void* p = NULL; + return posix_memalign(&p, alignment, size) == 0 ? p : NULL; +} +static void det_aligned_free(void* p) { free(p); } +#endif + +static float* offset_ptr(lw_x64_det_instance* instance, uint64_t offset) { + if (instance == NULL || instance->arena == NULL || offset > SIZE_MAX) return NULL; + return (float*)(void*)(instance->arena + (size_t)offset); +} + +/* Defined below; the shard machinery sits between these and the op loop. */ +static lw_scalar_binary_op binary_operation(uint16_t operation); +static void scalar_nhwc_conv_row_range(const lw_x64_det_conv_op* conv, const float* input, + float* output, uint32_t row_begin, uint32_t row_end, + const float* post_bias, const float* residual); +static void scalar_nhwc_conv(const lw_x64_det_conv_op* conv, const float* input, float* output, + const float* post_bias, const float* residual); +static void scalar_nhwc_batch_norm(const lw_x64_det_affine_op* affine, const float* input, + float* output); + +/* Threaded-run state carried through the op loop. NULL keeps the proven + * serial path (contract driver, test hooks). */ +typedef struct lw_x64_det_run_state { + lw_x64_det_instance* instance; + lw_thread_pool* pool; + uint32_t worker_count; +} lw_x64_det_run_state; + +/* Reference sharding thresholds (SimdPaddleOCR): dense/pointwise shard at 2M + * MACs, depthwise and the 1-channel probability map at 1M. Workers cap at the + * pool size (16). */ +#define LW_X64_DET_CONV_SHARD_MACS UINT64_C(2000000) +#define LW_X64_DET_DEPTHWISE_SHARD_MACS UINT64_C(1000000) +#define LW_X64_DET_ELEMENTWISE_SHARD_ELEMS UINT64_C(131072) +#define LW_X64_DET_CONCAT_SHARD_PIXELS UINT64_C(8192) + +static uint64_t conv_macs(const lw_x64_det_conv_op* conv) { + uint64_t macs = (uint64_t)conv->output_channels * conv->input_channels * + conv->kernel_h * conv->kernel_w * conv->output_height * conv->output_width; + /* Depthwise per-pixel work does not scale with the channel count. */ + if (conv->groups == conv->input_channels && conv->groups == conv->output_channels && + conv->groups != 1u) { + macs /= conv->input_channels; + } + return macs; +} + +static uint32_t det_shard_workers(const lw_x64_det_run_state* state, uint64_t macs, + uint64_t threshold) { + if (state == NULL || state->pool == NULL || state->worker_count <= 1u || + macs < threshold) { + return 1u; + } + return state->worker_count > LW_PARALLEL_MAX_WORKERS + ? LW_PARALLEL_MAX_WORKERS : state->worker_count; +} + +static lw_scalar_binary_op binary_operation(uint16_t operation); + +/* Shard worker context: one physical op, per-kind subranges. Whole-tile and + * whole-row partitioning keeps every output element computed by exactly one + * worker in the same summation order as the serial run (bit-identical). */ +typedef enum lw_x64_det_shard_kind { + LW_X64_DET_SHARD_POINTWISE_SPATIAL = 0, + LW_X64_DET_SHARD_DENSE_ROWS = 2, + LW_X64_DET_SHARD_DEPTHWISE_ROWS = 3, + LW_X64_DET_SHARD_CT16_ROWS = 4, + LW_X64_DET_SHARD_CTC1_ROWS = 5, + LW_X64_DET_SHARD_BINARY_SAME = 6, + LW_X64_DET_SHARD_BINARY_SCALAR = 7, + LW_X64_DET_SHARD_BINARY_CHANNEL = 8, + LW_X64_DET_SHARD_SIGMOID = 9, + LW_X64_DET_SHARD_CONCAT_NHWC = 10, + LW_X64_DET_SHARD_POOL_NCHW_CHANNELS = 11, + LW_X64_DET_SHARD_POOL_NHWC_ROWS = 12, + LW_X64_DET_SHARD_REDUCE_CHANNELS = 13 +} lw_x64_det_shard_kind; + +typedef struct lw_x64_det_shard_ctx { + uint16_t kind; + uint16_t pointwise_kernel; + const lw_x64_det_op* op; + lw_x64_det_instance* instance; + const float* input; + const float* input2; + float* output_base; + uint8_t binary_left_is_channel; + lw_nhwc_epilogue epilogue; + uint32_t items; + uint32_t dense_kc; + uint8_t* scratch_base; + uint64_t scratch_bytes; + lw_status statuses[LW_PARALLEL_MAX_WORKERS]; +} lw_x64_det_shard_ctx; + +static void det_shard_worker(void* opaque, uint32_t worker_index, uint32_t worker_count) { + lw_x64_det_shard_ctx* ctx = (lw_x64_det_shard_ctx*)opaque; + const lw_x64_det_conv_op* conv = &ctx->op->data.conv; + uint32_t begin; + uint32_t end; + ctx->statuses[worker_index] = LW_STATUS_OK; + switch (ctx->kind) { + case LW_X64_DET_SHARD_POINTWISE_SPATIAL: { + lw_nhwc_epilogue ep = ctx->epilogue; + begin = (uint32_t)(((uint64_t)ctx->items * worker_index) / worker_count); + end = (uint32_t)(((uint64_t)ctx->items * (worker_index + 1u)) / worker_count); + if (begin >= end) return; + /* The pointwise kernels index the residual by slice-local pixel, so + * advance it to this worker's first pixel (same offset as output). */ + if (ep.residual != NULL) ep.residual += (size_t)begin * conv->output_channels; + switch (ctx->pointwise_kernel) { + case LW_X64_DET_PW_4X16: + lw_avx2_fma_nhwc_pointwise_4x16_f32( + ctx->input + (size_t)begin * conv->input_channels, conv->packed_weights, + &ep, ctx->output_base + (size_t)begin * conv->output_channels, + end - begin, conv->input_channels, conv->output_channels); + break; + case LW_X64_DET_PW_3X32: + lw_avx2_fma_nhwc_pointwise_3x32_f32( + ctx->input + (size_t)begin * conv->input_channels, conv->packed_weights, + &ep, ctx->output_base + (size_t)begin * conv->output_channels, + end - begin, conv->input_channels, conv->output_channels); + break; + case LW_X64_DET_PW_2X32: + lw_avx2_fma_nhwc_pointwise_2x32_f32( + ctx->input + (size_t)begin * conv->input_channels, conv->packed_weights, + &ep, ctx->output_base + (size_t)begin * conv->output_channels, + end - begin, conv->input_channels, conv->output_channels); + break; + default: + lw_avx2_fma_nhwc_pointwise_f32( + ctx->input + (size_t)begin * conv->input_channels, conv->packed_weights, + &ep, ctx->output_base + (size_t)begin * conv->output_channels, + end - begin, conv->input_channels, conv->output_channels); + break; + } + break; + } + case LW_X64_DET_SHARD_DENSE_ROWS: { + begin = (uint32_t)(((uint64_t)ctx->items * worker_index) / worker_count); + end = (uint32_t)(((uint64_t)ctx->items * (worker_index + 1u)) / worker_count); + if (begin >= end) return; + lw_nhwc_dense_desc desc; + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; + desc.input_channels = conv->input_channels; + desc.input_height = conv->input_height; + desc.input_width = conv->input_width; + desc.output_channels = conv->output_channels; + desc.output_height = end - begin; + desc.output_width = conv->output_width; + desc.kernel_h = conv->kernel_h; + desc.kernel_w = conv->kernel_w; + desc.stride_h = conv->stride_h; + desc.stride_w = conv->stride_w; + desc.pad_top = conv->pad_top; + desc.pad_left = conv->pad_left; + desc.pad_bottom = conv->pad_bottom; + desc.pad_right = conv->pad_right; + desc.dense_kc = ctx->dense_kc; + desc.output_row_offset = begin; + { + lw_nhwc_epilogue ep = ctx->epilogue; + /* The dense kernel indexes the residual by slice-local row, so + * advance it to this worker's first row (same offset as output). */ + if (ep.residual != NULL) { + ep.residual += (size_t)begin * conv->output_width * conv->output_channels; + } + ctx->statuses[worker_index] = lw_avx2_fma_nhwc_dense_f32( + ctx->input, conv->packed_weights, &ep, + ctx->output_base + (size_t)begin * conv->output_width * conv->output_channels, + &desc, ctx->scratch_base + (size_t)worker_index * ctx->scratch_bytes, + ctx->scratch_bytes); + } + if (ctx->statuses[worker_index] != LW_STATUS_OK) { + scalar_nhwc_conv_row_range(conv, ctx->input, ctx->output_base, begin, end, + ctx->epilogue.post_bias, ctx->epilogue.residual); + ctx->statuses[worker_index] = LW_STATUS_OK; + } + break; + } + case LW_X64_DET_SHARD_DEPTHWISE_ROWS: { + begin = (uint32_t)(((uint64_t)ctx->items * worker_index) / worker_count); + end = (uint32_t)(((uint64_t)ctx->items * (worker_index + 1u)) / worker_count); + if (begin >= end) return; + lw_nhwc_depthwise_desc desc; + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; + desc.channels = conv->input_channels; + desc.input_height = conv->input_height; + desc.input_width = conv->input_width; + desc.output_height = end - begin; + desc.output_width = conv->output_width; + desc.kernel_h = conv->kernel_h; + desc.kernel_w = conv->kernel_w; + desc.stride_h = conv->stride_h; + desc.stride_w = conv->stride_w; + desc.pad_top = conv->pad_top; + desc.pad_left = conv->pad_left; + desc.pad_bottom = conv->pad_bottom; + desc.pad_right = conv->pad_right; + desc.output_row_offset = begin; + ctx->statuses[worker_index] = lw_avx2_fma_nhwc_depthwise_f32( + ctx->input, conv->packed_weights, ctx->epilogue.bias, + ctx->output_base + (size_t)begin * conv->output_width * conv->output_channels, + &desc); + if (ctx->statuses[worker_index] != LW_STATUS_OK) { + scalar_nhwc_conv_row_range(conv, ctx->input, ctx->output_base, begin, end, + NULL, NULL); + ctx->statuses[worker_index] = LW_STATUS_OK; + } + break; + } + case LW_X64_DET_SHARD_CT16_ROWS: + case LW_X64_DET_SHARD_CTC1_ROWS: { + const lw_x64_det_conv_transpose_op* ct = &ctx->op->data.conv_transpose; + begin = (uint32_t)(((uint64_t)ctx->items * worker_index) / worker_count); + end = (uint32_t)(((uint64_t)ctx->items * (worker_index + 1u)) / worker_count); + if (begin >= end) return; + lw_nhwc_convtranspose_desc desc; + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; + desc.input_channels = (uint32_t)ct->input_dimensions[1]; + desc.input_height = end - begin; + desc.input_width = (uint32_t)ct->input_dimensions[3]; + desc.output_channels = (uint32_t)ct->output_dimensions[1]; + desc.output_height = (end - begin) * 2u; + desc.output_width = (uint32_t)ct->output_dimensions[3]; + if (ctx->kind == LW_X64_DET_SHARD_CTC1_ROWS) { + ctx->statuses[worker_index] = lw_avx2_fma_nhwc_convtranspose2x2_s2_c1_f32( + ctx->input + (size_t)begin * desc.input_width * desc.input_channels, + ct->weights, &ctx->epilogue, + ctx->output_base + (size_t)(begin * 2u) * desc.output_width * + desc.output_channels, &desc); + } else { + ctx->statuses[worker_index] = lw_avx2_fma_nhwc_convtranspose2x2_s2_f32( + ctx->input + (size_t)begin * desc.input_width * desc.input_channels, + ct->packed_weights, &ctx->epilogue, + ctx->output_base + (size_t)(begin * 2u) * desc.output_width * + desc.output_channels, &desc); + } + break; + } + case LW_X64_DET_SHARD_BINARY_SAME: + case LW_X64_DET_SHARD_BINARY_SCALAR: + case LW_X64_DET_SHARD_SIGMOID: { + uint64_t element_begin = ((uint64_t)ctx->items * worker_index) / worker_count; + uint64_t element_end = ((uint64_t)ctx->items * (worker_index + 1u)) / worker_count; + if (element_begin >= element_end) return; + if (ctx->kind == LW_X64_DET_SHARD_SIGMOID) { + ctx->statuses[worker_index] = lw_scalar_sigmoid_f32( + ctx->input + element_begin, ctx->output_base + element_begin, + element_end - element_begin); + } else if (ctx->kind == LW_X64_DET_SHARD_BINARY_SCALAR) { + lw_avx2_binary_right_scalar_f32( + binary_operation(ctx->op->data.binary.operation), + ctx->input + element_begin, ctx->op->data.binary.right_constant[0], + ctx->output_base + element_begin, element_end - element_begin); + } else { + lw_avx2_binary_contiguous_f32( + binary_operation(ctx->op->data.binary.operation), + ctx->input + element_begin, ctx->input2 + element_begin, + ctx->output_base + element_begin, element_end - element_begin); + } + break; + } + case LW_X64_DET_SHARD_BINARY_CHANNEL: { + uint32_t channels = ctx->op->data.binary.channels; + begin = (uint32_t)(((uint64_t)ctx->items * worker_index) / worker_count); + end = (uint32_t)(((uint64_t)ctx->items * (worker_index + 1u)) / worker_count); + if (begin >= end) return; + lw_avx2_binary_channel_f32(binary_operation(ctx->op->data.binary.operation), + ctx->input + (size_t)begin * channels, ctx->input2, + ctx->output_base + (size_t)begin * channels, + (uint64_t)(end - begin), channels, + ctx->binary_left_is_channel); + break; + } + case LW_X64_DET_SHARD_CONCAT_NHWC: { + const float* inputs[LWM_V0_MAX_NODE_INPUTS]; + uint32_t input_slot; + uint32_t output_channels = 0u; + uint64_t pixel; + begin = (uint32_t)(((uint64_t)ctx->items * worker_index) / worker_count); + end = (uint32_t)(((uint64_t)ctx->items * (worker_index + 1u)) / worker_count); + if (begin >= end) return; + for (input_slot = 0u; input_slot < ctx->op->data.concat.input_count; ++input_slot) { + inputs[input_slot] = + offset_ptr(ctx->instance, ctx->op->data.concat.input_offsets[input_slot]); + if (inputs[input_slot] == NULL) { + ctx->statuses[worker_index] = LW_STATUS_INVALID_ARGUMENT; + return; + } + output_channels += (uint32_t)ctx->op->data.concat.input_dimensions[input_slot][1]; + } + for (pixel = begin; pixel < end; ++pixel) { + float* destination = ctx->output_base + (size_t)pixel * output_channels; + for (input_slot = 0u; input_slot < ctx->op->data.concat.input_count; ++input_slot) { + uint32_t channels = + (uint32_t)ctx->op->data.concat.input_dimensions[input_slot][1]; + memcpy(destination, inputs[input_slot] + (size_t)pixel * channels, + (size_t)channels * sizeof(float)); + destination += channels; + } + } + break; + } + case LW_X64_DET_SHARD_POOL_NCHW_CHANNELS: { + int32_t input_dims[4]; + int32_t output_dims[4]; + size_t input_slice; + size_t output_slice; + begin = (uint32_t)(((uint64_t)ctx->items * worker_index) / worker_count); + end = (uint32_t)(((uint64_t)ctx->items * (worker_index + 1u)) / worker_count); + if (begin >= end) return; + memcpy(input_dims, ctx->op->data.pool.input_dimensions, sizeof(input_dims)); + memcpy(output_dims, ctx->op->data.pool.output_dimensions, sizeof(output_dims)); + input_slice = (size_t)input_dims[2] * (size_t)input_dims[3]; + output_slice = (size_t)output_dims[2] * (size_t)output_dims[3]; + input_dims[1] = (int32_t)(end - begin); + output_dims[1] = (int32_t)(end - begin); + ctx->statuses[worker_index] = ctx->op->kind == LW_X64_DET_OP_MAX_POOL + ? lw_scalar_max_pool2d_f32(ctx->input + (size_t)begin * input_slice, + ctx->output_base + (size_t)begin * output_slice, + input_dims, output_dims, ctx->op->data.pool.kernel, + ctx->op->data.pool.strides, ctx->op->data.pool.pads, 0u) + : lw_scalar_average_pool2d_f32(ctx->input + (size_t)begin * input_slice, + ctx->output_base + (size_t)begin * output_slice, + input_dims, output_dims, ctx->op->data.pool.kernel, + ctx->op->data.pool.strides, ctx->op->data.pool.pads, 0u, + ctx->op->data.pool.count_include_pad); + break; + } + case LW_X64_DET_SHARD_POOL_NHWC_ROWS: { + uint32_t channels = (uint32_t)ctx->op->data.pool.input_dimensions[3]; + uint32_t input_width = (uint32_t)ctx->op->data.pool.input_dimensions[2]; + uint32_t output_width = (uint32_t)ctx->op->data.pool.output_dimensions[2]; + uint32_t stride_h = (uint32_t)ctx->op->data.pool.strides[0]; + uint32_t pad_top = (uint32_t)ctx->op->data.pool.pads[0]; + const float* input = ctx->input; + uint32_t input_height = (uint32_t)ctx->op->data.pool.input_dimensions[1]; + uint32_t pad = pad_top; + begin = (uint32_t)(((uint64_t)ctx->items * worker_index) / worker_count); + end = (uint32_t)(((uint64_t)ctx->items * (worker_index + 1u)) / worker_count); + if (begin >= end) return; + if (begin > 0u) { + uint32_t row_offset = begin * stride_h - pad_top; + input = ctx->input + (size_t)row_offset * input_width * channels; + input_height -= row_offset; + pad = 0u; + } + lw_avx2_nhwc_pool_f32( + input, ctx->output_base + (size_t)begin * output_width * channels, 1u, + input_height, input_width, end - begin, output_width, channels, + (uint32_t)ctx->op->data.pool.kernel[0], (uint32_t)ctx->op->data.pool.kernel[1], + stride_h, (uint32_t)ctx->op->data.pool.strides[1], pad, + (uint32_t)ctx->op->data.pool.pads[1], ctx->op->data.pool.count_include_pad, + ctx->op->data.pool.is_max); + break; + } + case LW_X64_DET_SHARD_REDUCE_CHANNELS: { + uint32_t channels = ctx->op->data.reduce.channels; + uint32_t height = ctx->op->data.reduce.height; + uint32_t width = ctx->op->data.reduce.width; + begin = (uint32_t)(((uint64_t)ctx->items * worker_index) / worker_count); + end = (uint32_t)(((uint64_t)ctx->items * (worker_index + 1u)) / worker_count); + if (begin >= end) return; + if (ctx->op->data.reduce.layout == LW_X64_DET_LAYOUT_NCHW) { + uint32_t channel; + for (channel = begin; channel < end; ++channel) { + const float* source = ctx->input + (size_t)channel * height * width; + float sum = 0.0f; + uint32_t spatial; + for (spatial = 0u; spatial < height * width; ++spatial) { + sum += source[spatial]; + } + ctx->output_base[channel] = sum / (float)(height * width); + } + } else { + lw_avx2_nhwc_reduce_mean_hw_strided_f32( + ctx->input + begin, ctx->output_base + begin, + ctx->op->data.reduce.batch, height, width, end - begin, + channels, channels); + } + break; + } + default: + ctx->statuses[worker_index] = LW_STATUS_UNSUPPORTED; + break; + } +} + +static lw_status run_sharded_conv(lw_x64_det_instance* instance, const lw_x64_det_op* op, + const lw_x64_det_run_state* state, + uint16_t shard_kind, uint32_t items, + lw_nhwc_epilogue epilogue) { + lw_x64_det_shard_ctx ctx; + uint64_t macs; + uint64_t threshold = LW_X64_DET_CONV_SHARD_MACS; + uint32_t workers; + uint32_t w; + memset(&ctx, 0, sizeof(ctx)); + ctx.kind = shard_kind; + ctx.op = op; + ctx.epilogue = epilogue; + ctx.items = items; + ctx.dense_kc = op->data.conv.dense_kc; + ctx.pointwise_kernel = op->data.conv.pointwise_kernel; + switch (shard_kind) { + case LW_X64_DET_SHARD_POINTWISE_SPATIAL: + case LW_X64_DET_SHARD_DENSE_ROWS: + macs = conv_macs(&op->data.conv); + ctx.input = offset_ptr(instance, op->data.conv.input_offset); + ctx.output_base = offset_ptr(instance, op->data.conv.output_offset); + ctx.scratch_bytes = op->data.conv.scratch_bytes; + ctx.scratch_base = instance->shard_scratch; + break; + case LW_X64_DET_SHARD_DEPTHWISE_ROWS: + threshold = LW_X64_DET_DEPTHWISE_SHARD_MACS; + macs = conv_macs(&op->data.conv); + ctx.input = offset_ptr(instance, op->data.conv.input_offset); + ctx.output_base = offset_ptr(instance, op->data.conv.output_offset); + break; + case LW_X64_DET_SHARD_CT16_ROWS: + case LW_X64_DET_SHARD_CTC1_ROWS: + macs = (uint64_t)op->data.conv_transpose.input_dimensions[1] * + op->data.conv_transpose.output_dimensions[1] * 4u * + op->data.conv_transpose.output_dimensions[2] * + op->data.conv_transpose.output_dimensions[3]; + threshold = shard_kind == LW_X64_DET_SHARD_CTC1_ROWS + ? LW_X64_DET_DEPTHWISE_SHARD_MACS : LW_X64_DET_CONV_SHARD_MACS; + ctx.input = offset_ptr(instance, op->data.conv_transpose.input_offset); + ctx.output_base = offset_ptr(instance, op->data.conv_transpose.output_offset); + break; + default: + return LW_STATUS_UNSUPPORTED; + } + if (ctx.input == NULL || ctx.output_base == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (shard_kind == LW_X64_DET_SHARD_DENSE_ROWS && + (ctx.scratch_base == NULL || ctx.scratch_bytes == 0u)) { + return LW_STATUS_INVALID_ARGUMENT; + } + workers = det_shard_workers(state, macs, threshold); + if (workers <= 1u) return LW_STATUS_UNSUPPORTED; + lw_thread_pool_run(state->pool, workers, det_shard_worker, &ctx); + for (w = 0u; w < workers; ++w) { + if (ctx.statuses[w] != LW_STATUS_OK) return ctx.statuses[w]; + } + return LW_STATUS_OK; +} + +/* Elementwise-style sharding: every output element is computed independently + * by the same per-element kernel as the serial path, so splitting the + * contiguous range keeps results bit-identical. Returns LW_STATUS_UNSUPPORTED + * when the op is too small or no pool is available and the caller must run + * the serial kernel instead. */ +static lw_status run_sharded_elementwise(lw_x64_det_instance* instance, + const lw_x64_det_op* op, + const lw_x64_det_run_state* state, + uint16_t shard_kind, const float* input, + const float* input2, float* output, uint64_t items, + uint64_t threshold, int binary_left_is_channel) { + lw_x64_det_shard_ctx ctx; + uint32_t workers; + uint32_t w; + if (items > UINT32_MAX) return LW_STATUS_UNSUPPORTED; + workers = det_shard_workers(state, items, threshold); + if (workers <= 1u) return LW_STATUS_UNSUPPORTED; + memset(&ctx, 0, sizeof(ctx)); + ctx.kind = shard_kind; + ctx.op = op; + ctx.instance = instance; + ctx.input = input; + ctx.input2 = input2; + ctx.output_base = output; + ctx.items = (uint32_t)items; + ctx.binary_left_is_channel = (uint8_t)binary_left_is_channel; + lw_thread_pool_run(state->pool, workers, det_shard_worker, &ctx); + for (w = 0u; w < workers; ++w) { + if (ctx.statuses[w] != LW_STATUS_OK) return ctx.statuses[w]; + } + return LW_STATUS_OK; +} + +static lw_scalar_binary_op binary_operation(uint16_t operation) { + switch (operation) { + case LW_OP_ADD: return LW_SCALAR_BINARY_ADD; + case LW_OP_MUL: return LW_SCALAR_BINARY_MUL; + case LW_OP_DIV: return LW_SCALAR_BINARY_DIV; + case LW_OP_SUB: return LW_SCALAR_BINARY_SUB; + default: return LW_SCALAR_BINARY_POW; + } +} + +static void scalar_nhwc_conv_row_range(const lw_x64_det_conv_op* conv, const float* input, + float* output, uint32_t row_begin, uint32_t row_end, + const float* post_bias, const float* residual) { + uint32_t oy, ox, oc, ky, kx, ic; + for (oy = row_begin; oy < row_end; ++oy) { + for (ox = 0u; ox < conv->output_width; ++ox) { + for (oc = 0u; oc < conv->output_channels; ++oc) { + float sum = conv->bias == NULL ? 0.0f : conv->bias[oc]; + for (ky = 0u; ky < conv->kernel_h; ++ky) { + int32_t iy = (int32_t)(oy * conv->stride_h + ky) - (int32_t)conv->pad_top; + if (iy < 0 || iy >= (int32_t)conv->input_height) continue; + for (kx = 0u; kx < conv->kernel_w; ++kx) { + int32_t ix = (int32_t)(ox * conv->stride_w + kx) - (int32_t)conv->pad_left; + if (ix < 0 || ix >= (int32_t)conv->input_width) continue; + for (ic = 0u; ic < conv->input_channels; ++ic) { + size_t input_index = ((size_t)iy * conv->input_width + (size_t)ix) * conv->input_channels + ic; + size_t weight_index; + if (conv->groups == conv->input_channels) { + if (ic != oc) continue; + weight_index = (((size_t)oc * conv->kernel_h + ky) * conv->kernel_w) + kx; + } else { + weight_index = ((((size_t)oc * conv->input_channels + ic) * conv->kernel_h + ky) * conv->kernel_w) + kx; + } + sum += input[input_index] * conv->original_weights[weight_index]; + } + } + } + if (post_bias != NULL) sum += post_bias[oc]; + if (residual != NULL) { + sum += residual[((size_t)oy * conv->output_width + ox) * + conv->output_channels + oc]; + } + if (conv->activation == LW_NHWC_ACT_RELU && sum < 0.0f) sum = 0.0f; + else if (conv->activation == LW_NHWC_ACT_GELU) sum = lw_nhwc_gelu_scalar_exact_f32(sum); + output[((size_t)oy * conv->output_width + ox) * conv->output_channels + oc] = sum; + } + } + } +} + +static void scalar_nhwc_conv(const lw_x64_det_conv_op* conv, const float* input, + float* output, const float* post_bias, const float* residual) { + scalar_nhwc_conv_row_range(conv, input, output, 0u, conv->output_height, + post_bias, residual); +} + +static void scalar_nhwc_batch_norm(const lw_x64_det_affine_op* affine, const float* input, + float* output) { + uint32_t pixel; + for (pixel = 0u; pixel < affine->pixels; ++pixel) { + uint32_t channel; + for (channel = 0u; channel < affine->channels; ++channel) { + float factor = affine->scale[channel] / sqrtf(affine->variance[channel] + affine->epsilon); + size_t index = affine->channel_major + ? (size_t)channel * affine->pixels + pixel + : (size_t)pixel * affine->channels + channel; + float value = input[index]; + output[index] = (value - affine->mean[channel]) * factor + affine->bias[channel]; + } + } +} + +static lw_status run_sharded_conv(lw_x64_det_instance* instance, const lw_x64_det_op* op, + const lw_x64_det_run_state* state, + uint16_t shard_kind, uint32_t items, + lw_nhwc_epilogue epilogue); + +static lw_status execute_op(lw_x64_det_instance* instance, const lw_x64_det_op* op, + const lw_x64_det_run_state* state, lw_error* error) { + (void)error; + float* input; + float* output; + if (op == NULL) return LW_STATUS_INVALID_ARGUMENT; + switch (op->kind) { + case LW_X64_DET_OP_CONV_NCHW: { + int32_t input_dimensions[4] = { + 1, (int32_t)op->data.conv.input_channels, + (int32_t)op->data.conv.input_height, + (int32_t)op->data.conv.input_width + }; + int32_t output_dimensions[4] = { + 1, (int32_t)op->data.conv.output_channels, + (int32_t)op->data.conv.output_height, + (int32_t)op->data.conv.output_width + }; + input = offset_ptr(instance, op->data.conv.input_offset); + output = offset_ptr(instance, op->data.conv.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + switch (op->data.conv.nchw_conv_kernel) { + case LW_X64_DET_NCHW_CONV_POINTWISE_PACKED: + if (op->data.conv.nchw_pointwise_kernel == LW_X64_DET_NCHW_PW_FMA8) { + lw_avx2_fma_packed_conv1x1_8x8_f32( + input, op->data.conv.packed_weights, op->data.conv.bias, + output, input_dimensions, output_dimensions); + } else { + lw_avx2_fma_packed_conv1x1_f32( + input, op->data.conv.packed_weights, op->data.conv.bias, + output, input_dimensions, output_dimensions); + } + return LW_STATUS_OK; + case LW_X64_DET_NCHW_CONV_3X3S2: + lw_avx2_conv3x3_stride2_pad1_f32( + input, op->data.conv.original_weights, op->data.conv.bias, + output, input_dimensions, output_dimensions); + return LW_STATUS_OK; + case LW_X64_DET_NCHW_CONV_3X3_UNIT: + lw_avx2_conv3x3_unit_pad1_f32( + input, op->data.conv.original_weights, op->data.conv.bias, + output, input_dimensions, output_dimensions); + return LW_STATUS_OK; + case LW_X64_DET_NCHW_CONV_2X2_PADEND1: + lw_avx2_conv2x2_unit_pad_end1_f32( + input, op->data.conv.original_weights, op->data.conv.bias, + output, input_dimensions, output_dimensions); + return LW_STATUS_OK; + case LW_X64_DET_NCHW_CONV_DEPTHWISE3X3: + lw_avx2_depthwise_conv3x3_unit_pad1_f32( + input, op->data.conv.original_weights, op->data.conv.bias, + output, input_dimensions); + return LW_STATUS_OK; + case LW_X64_DET_NCHW_CONV_DEPTHWISE3X3_S2X1: + lw_avx2_depthwise_conv3x3_stride2x1_pad1_f32( + input, op->data.conv.original_weights, op->data.conv.bias, + output, input_dimensions, output_dimensions); + return LW_STATUS_OK; + case LW_X64_DET_NCHW_CONV_DEPTHWISE5X5: + lw_avx2_depthwise_conv5x5_unit_pad2_f32( + input, op->data.conv.original_weights, op->data.conv.bias, + output, input_dimensions); + return LW_STATUS_OK; + case LW_X64_DET_NCHW_CONV_SCALAR: + default: { + int32_t weight_dimensions[4] = { + (int32_t)op->data.conv.output_channels, + op->data.conv.groups == op->data.conv.input_channels + ? 1 : (int32_t)op->data.conv.input_channels, + (int32_t)op->data.conv.weight_h, (int32_t)op->data.conv.weight_w + }; + int32_t kernel[2] = { (int32_t)op->data.conv.kernel_h, (int32_t)op->data.conv.kernel_w }; + int32_t strides[2] = { (int32_t)op->data.conv.stride_h, (int32_t)op->data.conv.stride_w }; + int32_t dilations[2] = { 1, 1 }; + int32_t pads[4] = { (int32_t)op->data.conv.pad_top, (int32_t)op->data.conv.pad_left, + (int32_t)op->data.conv.pad_bottom, (int32_t)op->data.conv.pad_right }; + lw_status fallback_status = lw_scalar_conv2d_f32( + input, op->data.conv.original_weights, op->data.conv.bias, + op->data.conv.bias != NULL ? op->data.conv.output_channels : 0u, output, + input_dimensions, weight_dimensions, output_dimensions, kernel, strides, + dilations, pads, op->data.conv.groups); + if (fallback_status != LW_STATUS_OK) { + lw_set_error(error, fallback_status, "NCHW conv scalar fallback failed"); + return fallback_status; + } + return LW_STATUS_OK; + } + } + } + case LW_X64_DET_OP_POINTWISE: { + lw_nhwc_epilogue ep = { NULL, NULL, op->data.conv.activation, 0u, 0.0f, 0.0f, NULL }; + uint32_t pixels = op->data.conv.input_height * op->data.conv.input_width; + uint32_t workers = det_shard_workers(state, conv_macs(&op->data.conv), + LW_X64_DET_CONV_SHARD_MACS); + input = offset_ptr(instance, op->data.conv.input_offset); + output = offset_ptr(instance, op->data.conv.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + ep.bias = op->data.conv.bias; + ep.post_bias = op->data.conv.post_bias; + if (op->data.conv.has_residual) { + ep.residual = offset_ptr(instance, op->data.conv.residual_offset); + if (ep.residual == NULL) return LW_STATUS_INVALID_ARGUMENT; + } + if (workers > 1u) { + /* The pointwise kernels write tightly packed pixel rows. Shard by + * pixels so each worker keeps the full NHWC output-channel stride. */ + lw_status status = run_sharded_conv( + instance, op, state, LW_X64_DET_SHARD_POINTWISE_SPATIAL, pixels, ep); + if (status != LW_STATUS_OK) return status; + return LW_STATUS_OK; + } +#if defined(LW_WASM_COMPILED_DET) + lw_rec_backend_kernels_current()->pointwise( + input, op->data.conv.packed_weights, &ep, output, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); +#else + switch (op->data.conv.pointwise_kernel) { + case LW_X64_DET_PW_4X16: + lw_avx2_fma_nhwc_pointwise_4x16_f32( + input, op->data.conv.packed_weights, &ep, output, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); + break; + case LW_X64_DET_PW_3X32: + lw_avx2_fma_nhwc_pointwise_3x32_f32( + input, op->data.conv.packed_weights, &ep, output, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); + break; + case LW_X64_DET_PW_2X32: + lw_avx2_fma_nhwc_pointwise_2x32_f32( + input, op->data.conv.packed_weights, &ep, output, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); + break; + default: + lw_avx2_fma_nhwc_pointwise_f32( + input, op->data.conv.packed_weights, &ep, output, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); + break; + } +#endif + return LW_STATUS_OK; + } + case LW_X64_DET_OP_DENSE: { + lw_nhwc_dense_desc desc; + lw_nhwc_epilogue ep; + lw_status status; + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; + desc.input_channels = op->data.conv.input_channels; + desc.input_height = op->data.conv.input_height; + desc.input_width = op->data.conv.input_width; + desc.output_channels = op->data.conv.output_channels; + desc.output_height = op->data.conv.output_height; + desc.output_width = op->data.conv.output_width; + desc.kernel_h = op->data.conv.kernel_h; + desc.kernel_w = op->data.conv.kernel_w; + desc.stride_h = op->data.conv.stride_h; + desc.stride_w = op->data.conv.stride_w; + desc.pad_top = op->data.conv.pad_top; + desc.pad_left = op->data.conv.pad_left; + desc.pad_bottom = op->data.conv.pad_bottom; + desc.pad_right = op->data.conv.pad_right; + desc.dense_kc = op->data.conv.dense_kc; + memset(&ep, 0, sizeof(ep)); + ep.bias = op->data.conv.bias; + ep.activation = op->data.conv.activation; + ep.post_bias = op->data.conv.post_bias; + input = offset_ptr(instance, op->data.conv.input_offset); + output = offset_ptr(instance, op->data.conv.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.conv.has_residual) { + ep.residual = offset_ptr(instance, op->data.conv.residual_offset); + if (ep.residual == NULL) return LW_STATUS_INVALID_ARGUMENT; + } + if (det_shard_workers(state, conv_macs(&op->data.conv), + LW_X64_DET_CONV_SHARD_MACS) > 1u && + op->data.conv.output_height > 1u) { + return run_sharded_conv(instance, op, state, LW_X64_DET_SHARD_DENSE_ROWS, + op->data.conv.output_height, ep); + } + status = lw_avx2_fma_nhwc_dense_f32( + input, op->data.conv.packed_weights, &ep, output, &desc, + instance->scratch, op->data.conv.scratch_bytes); + if (status == LW_STATUS_OK) return LW_STATUS_OK; + scalar_nhwc_conv(&op->data.conv, input, output, ep.post_bias, ep.residual); + return LW_STATUS_OK; + } + case LW_X64_DET_OP_DEPTHWISE: { + lw_nhwc_depthwise_desc desc; + lw_status status; + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; + desc.channels = op->data.conv.input_channels; + desc.input_height = op->data.conv.input_height; + desc.input_width = op->data.conv.input_width; + desc.output_height = op->data.conv.output_height; + desc.output_width = op->data.conv.output_width; + desc.kernel_h = op->data.conv.kernel_h; + desc.kernel_w = op->data.conv.kernel_w; + desc.stride_h = op->data.conv.stride_h; + desc.stride_w = op->data.conv.stride_w; + desc.pad_top = op->data.conv.pad_top; + desc.pad_left = op->data.conv.pad_left; + desc.pad_bottom = op->data.conv.pad_bottom; + desc.pad_right = op->data.conv.pad_right; + input = offset_ptr(instance, op->data.conv.input_offset); + output = offset_ptr(instance, op->data.conv.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (det_shard_workers(state, conv_macs(&op->data.conv), + LW_X64_DET_DEPTHWISE_SHARD_MACS) > 1u && + op->data.conv.output_height > 1u) { + lw_nhwc_epilogue ep = { op->data.conv.bias, NULL, 0u, 0u, 0.0f, 0.0f, NULL }; + return run_sharded_conv(instance, op, state, LW_X64_DET_SHARD_DEPTHWISE_ROWS, + op->data.conv.output_height, ep); + } + status = lw_avx2_fma_nhwc_depthwise_f32(input, op->data.conv.packed_weights, + op->data.conv.bias, output, &desc); + if (status == LW_STATUS_OK) return LW_STATUS_OK; + scalar_nhwc_conv(&op->data.conv, input, output, NULL, NULL); + return LW_STATUS_OK; + } + case LW_X64_DET_OP_LAYOUT_CONVERT: + input = offset_ptr(instance, op->data.convert.input_offset); + output = offset_ptr(instance, op->data.convert.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.convert.to_nhwc) { + lw_x64_fast_nchw_to_nhwc(input, output, op->data.convert.batch, + op->data.convert.channels, op->data.convert.height, + op->data.convert.width); + } else { + lw_x64_fast_nhwc_to_nchw(input, output, op->data.convert.batch, + op->data.convert.channels, op->data.convert.height, + op->data.convert.width); + } + return LW_STATUS_OK; + case LW_X64_DET_OP_CONV_TRANSPOSE_NCHW: + case LW_X64_DET_OP_CONV_TRANSPOSE_NHWC: + case LW_X64_DET_OP_CONV_TRANSPOSE_NHWC_C1: + input = offset_ptr(instance, op->data.conv_transpose.input_offset); + output = offset_ptr(instance, op->data.conv_transpose.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->kind == LW_X64_DET_OP_CONV_TRANSPOSE_NCHW) { + lw_scalar_conv_transpose2x2_stride2_f32( + input, op->data.conv_transpose.weights, op->data.conv_transpose.bias, + output, op->data.conv_transpose.input_dimensions, + op->data.conv_transpose.output_dimensions); + return LW_STATUS_OK; + } + { + lw_nhwc_convtranspose_desc desc; + lw_nhwc_epilogue epilogue; + uint64_t ct_macs = (uint64_t)op->data.conv_transpose.input_dimensions[1] * + op->data.conv_transpose.output_dimensions[1] * 4u * + op->data.conv_transpose.output_dimensions[2] * + op->data.conv_transpose.output_dimensions[3]; + memset(&epilogue, 0, sizeof(epilogue)); + epilogue.bias = op->data.conv_transpose.bias; + if (det_shard_workers( + state, ct_macs, + op->kind == LW_X64_DET_OP_CONV_TRANSPOSE_NHWC_C1 + ? LW_X64_DET_DEPTHWISE_SHARD_MACS : LW_X64_DET_CONV_SHARD_MACS) > 1u && + op->data.conv_transpose.input_dimensions[2] > 1) { + return run_sharded_conv( + instance, op, state, + op->kind == LW_X64_DET_OP_CONV_TRANSPOSE_NHWC_C1 + ? LW_X64_DET_SHARD_CTC1_ROWS : LW_X64_DET_SHARD_CT16_ROWS, + (uint32_t)op->data.conv_transpose.input_dimensions[2], epilogue); + } + memset(&desc, 0, sizeof(desc)); + desc.batch = (uint32_t)op->data.conv_transpose.input_dimensions[0]; + desc.input_channels = (uint32_t)op->data.conv_transpose.input_dimensions[1]; + desc.input_height = (uint32_t)op->data.conv_transpose.input_dimensions[2]; + desc.input_width = (uint32_t)op->data.conv_transpose.input_dimensions[3]; + desc.output_channels = (uint32_t)op->data.conv_transpose.output_dimensions[1]; + desc.output_height = (uint32_t)op->data.conv_transpose.output_dimensions[2]; + desc.output_width = (uint32_t)op->data.conv_transpose.output_dimensions[3]; + if (op->kind == LW_X64_DET_OP_CONV_TRANSPOSE_NHWC_C1) { +#if defined(LW_WASM_COMPILED_DET) + return lw_wasm128_nhwc_convtranspose2x2_s2_c1_f32( + input, op->data.conv_transpose.weights, &epilogue, output, &desc); +#else + return lw_avx2_fma_nhwc_convtranspose2x2_s2_c1_f32( + input, op->data.conv_transpose.weights, &epilogue, output, &desc); +#endif + } +#if defined(LW_WASM_COMPILED_DET) + return lw_wasm128_nhwc_convtranspose2x2_s2_f32( + input, op->data.conv_transpose.packed_weights, &epilogue, output, &desc); +#else + return lw_avx2_fma_nhwc_convtranspose2x2_s2_f32( + input, op->data.conv_transpose.packed_weights, &epilogue, output, &desc); +#endif + } + case LW_X64_DET_OP_RESIZE_NCHW: + input = offset_ptr(instance, op->data.resize.input_offset); + output = offset_ptr(instance, op->data.resize.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + return lw_scalar_resize_nearest_f32( + input, output, op->data.resize.rank, op->data.resize.input_dimensions, + op->data.resize.output_dimensions, op->data.resize.scales); + case LW_X64_DET_OP_RESIZE_NHWC: + input = offset_ptr(instance, op->data.resize.input_offset); + output = offset_ptr(instance, op->data.resize.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + lw_avx2_nhwc_resize_nearest_f32( + input, output, (uint32_t)op->data.resize.input_dimensions[0], + (uint32_t)op->data.resize.input_dimensions[1], + (uint32_t)op->data.resize.input_dimensions[2], + (uint32_t)op->data.resize.input_dimensions[3], + (uint32_t)op->data.resize.output_dimensions[2], + (uint32_t)op->data.resize.output_dimensions[3]); + return LW_STATUS_OK; + case LW_X64_DET_OP_SIGMOID_NCHW: + input = offset_ptr(instance, op->data.unary.input_offset); + output = offset_ptr(instance, op->data.unary.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (state != NULL) { + lw_status sharded = run_sharded_elementwise( + instance, op, state, LW_X64_DET_SHARD_SIGMOID, input, NULL, output, + op->data.unary.element_count, LW_X64_DET_ELEMENTWISE_SHARD_ELEMS, 0); + if (sharded != LW_STATUS_UNSUPPORTED) return sharded; + } + return lw_scalar_sigmoid_f32(input, output, op->data.unary.element_count); + case LW_X64_DET_OP_CONCAT_NCHW: { + const float* inputs[LWM_V0_MAX_NODE_INPUTS]; + uint32_t input_slot; + for (input_slot = 0u; input_slot < op->data.concat.input_count; ++input_slot) { + inputs[input_slot] = offset_ptr(instance, op->data.concat.input_offsets[input_slot]); + if (inputs[input_slot] == NULL) return LW_STATUS_INVALID_ARGUMENT; + } + output = offset_ptr(instance, op->data.concat.output_offset); + if (output == NULL) return LW_STATUS_INVALID_ARGUMENT; + { + const int32_t* dimensions[LWM_V0_MAX_NODE_INPUTS]; + uint32_t ranks[LWM_V0_MAX_NODE_INPUTS]; + for (input_slot = 0u; input_slot < op->data.concat.input_count; ++input_slot) { + ranks[input_slot] = op->data.concat.input_ranks[input_slot]; + dimensions[input_slot] = op->data.concat.input_dimensions[input_slot]; + } + return lw_scalar_concat_f32(inputs, op->data.concat.input_count, ranks, dimensions, + output, op->data.concat.output_rank, + op->data.concat.output_dimensions, op->data.concat.axis); + } + } + case LW_X64_DET_OP_CONCAT_NHWC: { + /* Channel-axis concat in NHWC is per-pixel concatenated copies; all + * inputs share the spatial shape, so take it from the first input. */ + uint64_t pixel; + uint32_t input_slot; + const float* inputs[LWM_V0_MAX_NODE_INPUTS]; + uint64_t pixel_count; + uint32_t output_channels = 0u; + for (input_slot = 0u; input_slot < op->data.concat.input_count; ++input_slot) { + inputs[input_slot] = offset_ptr(instance, op->data.concat.input_offsets[input_slot]); + if (inputs[input_slot] == NULL) return LW_STATUS_INVALID_ARGUMENT; + output_channels += (uint32_t)op->data.concat.input_dimensions[input_slot][1]; + } + if (op->data.concat.input_count == 0u) return LW_STATUS_INVALID_ARGUMENT; + pixel_count = (uint64_t)(uint32_t)op->data.concat.input_dimensions[0][0] * + (uint64_t)(uint32_t)op->data.concat.input_dimensions[0][2] * + (uint64_t)(uint32_t)op->data.concat.input_dimensions[0][3]; + output = offset_ptr(instance, op->data.concat.output_offset); + if (output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (state != NULL) { + lw_status sharded = run_sharded_elementwise( + instance, op, state, LW_X64_DET_SHARD_CONCAT_NHWC, NULL, NULL, output, + pixel_count, LW_X64_DET_CONCAT_SHARD_PIXELS, 0); + if (sharded != LW_STATUS_UNSUPPORTED) return sharded; + } + for (pixel = 0u; pixel < pixel_count; ++pixel) { + float* destination = output + (size_t)pixel * output_channels; + for (input_slot = 0u; input_slot < op->data.concat.input_count; ++input_slot) { + uint32_t channels = + (uint32_t)op->data.concat.input_dimensions[input_slot][1]; + memcpy(destination, inputs[input_slot] + (size_t)pixel * channels, + (size_t)channels * sizeof(float)); + destination += channels; + } + } + return LW_STATUS_OK; + } + case LW_X64_DET_OP_AFFINE: + input = offset_ptr(instance, op->data.affine.input_offset); + output = offset_ptr(instance, op->data.affine.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.affine.channel_major && op->data.affine.pixels > 1u) { + lw_avx2_nchw_affine_f32(input, op->data.affine.mul, op->data.affine.add, + output, op->data.affine.channels, op->data.affine.pixels); + } else if (op->data.affine.channel_major) { + scalar_nhwc_batch_norm(&op->data.affine, input, output); + } else { +#if defined(LW_WASM_COMPILED_DET) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_affine_nhwc_f32(input, op->data.affine.mul, op->data.affine.add, + output, op->data.affine.pixels, + op->data.affine.channels); +#else + lw_avx2_nhwc_affine_f32(input, op->data.affine.mul, op->data.affine.add, + output, op->data.affine.pixels, op->data.affine.channels); +#endif + } + return LW_STATUS_OK; + case LW_X64_DET_OP_ADD: case LW_X64_DET_OP_MUL: case LW_X64_DET_OP_DIV: + input = op->data.binary.left_constant != NULL + ? (float*)(uintptr_t)op->data.binary.left_constant + : offset_ptr(instance, op->data.binary.left_offset); + output = offset_ptr(instance, op->data.binary.output_offset); + if (op->data.binary.layout == LW_X64_DET_LAYOUT_NCHW && + (op->data.binary.broadcast_kind == LW_X64_DET_BROADCAST_LEFT_CHANNEL || + op->data.binary.broadcast_kind == LW_X64_DET_BROADCAST_RIGHT_CHANNEL)) { + float* full = op->data.binary.broadcast_kind == LW_X64_DET_BROADCAST_LEFT_CHANNEL + ? offset_ptr(instance, op->data.binary.right_offset) : input; + float* channel = op->data.binary.broadcast_kind == LW_X64_DET_BROADCAST_LEFT_CHANNEL + ? (op->data.binary.left_constant != NULL + ? (float*)(uintptr_t)op->data.binary.left_constant + : offset_ptr(instance, op->data.binary.left_offset)) + : (op->data.binary.right_constant != NULL + ? (float*)(uintptr_t)op->data.binary.right_constant + : offset_ptr(instance, op->data.binary.right_offset)); + if (full == NULL || channel == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + lw_avx2_binary_channel_nchw_f32( + binary_operation(op->data.binary.operation), full, channel, output, + op->data.binary.pixels, op->data.binary.channels, + op->data.binary.broadcast_kind == LW_X64_DET_BROADCAST_LEFT_CHANNEL); + return LW_STATUS_OK; + } + if (op->data.binary.broadcast_kind == LW_X64_DET_BROADCAST_LEFT_CHANNEL) { + float* full = offset_ptr(instance, op->data.binary.right_offset); + float* channel = op->data.binary.left_constant != NULL + ? (float*)(uintptr_t)op->data.binary.left_constant + : offset_ptr(instance, op->data.binary.left_offset); + if (full == NULL || channel == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (state != NULL && op->data.binary.channels > 0u) { + uint64_t ch_threshold = LW_X64_DET_ELEMENTWISE_SHARD_ELEMS / + op->data.binary.channels; + lw_status sharded = run_sharded_elementwise( + instance, op, state, LW_X64_DET_SHARD_BINARY_CHANNEL, full, channel, output, + op->data.binary.pixels, ch_threshold < 2u ? 2u : ch_threshold, 1); + if (sharded != LW_STATUS_UNSUPPORTED) return sharded; + } +#if defined(LW_WASM_COMPILED_DET) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_binary_channel_nhwc_f32(binary_operation(op->data.binary.operation), + full, channel, output, op->data.binary.pixels, + op->data.binary.channels, 1); +#else + lw_avx2_binary_channel_f32(binary_operation(op->data.binary.operation), full, channel, + output, op->data.binary.pixels, op->data.binary.channels, 1); +#endif + return LW_STATUS_OK; + } + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.binary.broadcast_kind == LW_X64_DET_BROADCAST_RIGHT_SCALAR && + op->data.binary.right_constant != NULL) { + if (state != NULL) { + lw_status sharded = run_sharded_elementwise( + instance, op, state, LW_X64_DET_SHARD_BINARY_SCALAR, input, NULL, output, + op->data.binary.element_count, LW_X64_DET_ELEMENTWISE_SHARD_ELEMS, 0); + if (sharded != LW_STATUS_UNSUPPORTED) return sharded; + } +#if defined(LW_WASM_COMPILED_DET) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_binary_scalar_f32(binary_operation(op->data.binary.operation), input, + op->data.binary.right_constant[0], output, + op->data.binary.element_count, 0); +#else + lw_avx2_binary_right_scalar_f32(binary_operation(op->data.binary.operation), input, + op->data.binary.right_constant[0], output, + op->data.binary.element_count); +#endif + return LW_STATUS_OK; + } + if (op->data.binary.broadcast_kind == LW_X64_DET_BROADCAST_SAME) { + float* right = op->data.binary.right_constant != NULL + ? (float*)(uintptr_t)op->data.binary.right_constant + : offset_ptr(instance, op->data.binary.right_offset); + if (right == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (state != NULL) { + lw_status sharded = run_sharded_elementwise( + instance, op, state, LW_X64_DET_SHARD_BINARY_SAME, input, right, output, + op->data.binary.element_count, LW_X64_DET_ELEMENTWISE_SHARD_ELEMS, 0); + if (sharded != LW_STATUS_UNSUPPORTED) return sharded; + } +#if defined(LW_WASM_COMPILED_DET) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_binary_contiguous_f32(binary_operation(op->data.binary.operation), + input, right, output, op->data.binary.element_count); +#else + lw_avx2_binary_contiguous_f32(binary_operation(op->data.binary.operation), input, + right, output, op->data.binary.element_count); +#endif + return LW_STATUS_OK; + } + if (op->data.binary.broadcast_kind == LW_X64_DET_BROADCAST_RIGHT_CHANNEL) { + float* channel = op->data.binary.right_constant != NULL + ? (float*)(uintptr_t)op->data.binary.right_constant + : offset_ptr(instance, op->data.binary.right_offset); + if (channel == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (state != NULL && op->data.binary.channels > 0u) { + uint64_t ch_threshold = LW_X64_DET_ELEMENTWISE_SHARD_ELEMS / + op->data.binary.channels; + lw_status sharded = run_sharded_elementwise( + instance, op, state, LW_X64_DET_SHARD_BINARY_CHANNEL, input, channel, output, + op->data.binary.pixels, ch_threshold < 2u ? 2u : ch_threshold, 0); + if (sharded != LW_STATUS_UNSUPPORTED) return sharded; + } +#if defined(LW_WASM_COMPILED_DET) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_binary_channel_nhwc_f32(binary_operation(op->data.binary.operation), + input, channel, output, op->data.binary.pixels, + op->data.binary.channels, 0); +#else + lw_avx2_binary_channel_f32(binary_operation(op->data.binary.operation), input, channel, + output, op->data.binary.pixels, op->data.binary.channels, 0); +#endif + return LW_STATUS_OK; + } + if (op->data.binary.right_constant != NULL && op->data.binary.channels > 1u) { + uint64_t index; + const float* right = op->data.binary.right_constant; + for (index = 0u; index < op->data.binary.element_count; ++index) { + float lhs = input[index]; + float rhs = right[index % op->data.binary.channels]; + switch (op->data.binary.operation) { + case LW_OP_ADD: output[index] = lhs + rhs; break; + case LW_OP_MUL: output[index] = lhs * rhs; break; + case LW_OP_DIV: output[index] = lhs / rhs; break; + default: output[index] = lhs - rhs; break; + } + } + return LW_STATUS_OK; + } + return LW_STATUS_UNSUPPORTED; + case LW_X64_DET_OP_RELU: case LW_X64_DET_OP_ERF: case LW_X64_DET_OP_GELU: + case LW_X64_DET_OP_HARD_SIGMOID: + input = offset_ptr(instance, op->data.unary.input_offset); + output = offset_ptr(instance, op->data.unary.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->kind == LW_X64_DET_OP_RELU) { +#if defined(LW_WASM_COMPILED_DET) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_relu_f32(input, output, op->data.unary.element_count); +#else + lw_avx2_relu_contiguous_f32(input, output, op->data.unary.element_count); +#endif + } else if (op->kind == LW_X64_DET_OP_ERF) { + lw_avx2_erf_f32(input, output, op->data.unary.element_count); + } else if (op->kind == LW_X64_DET_OP_GELU) { + lw_avx2_gelu_f32(input, output, op->data.unary.element_count); + } else { + lw_avx2_hard_sigmoid_contiguous_f32(input, output, op->data.unary.element_count, + op->data.unary.alpha, op->data.unary.beta); + } + return LW_STATUS_OK; + case LW_X64_DET_OP_REDUCE_MEAN: + input = offset_ptr(instance, op->data.reduce.input_offset); + output = offset_ptr(instance, op->data.reduce.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (state != NULL && op->data.reduce.channels > 1u && + (uint64_t)op->data.reduce.batch * op->data.reduce.channels * + op->data.reduce.height * op->data.reduce.width >= + LW_X64_DET_ELEMENTWISE_SHARD_ELEMS) { + lw_status sharded = run_sharded_elementwise( + instance, op, state, LW_X64_DET_SHARD_REDUCE_CHANNELS, input, NULL, + output, op->data.reduce.channels, 2u, 0); + if (sharded != LW_STATUS_UNSUPPORTED) return sharded; + } + if (op->data.reduce.layout == LW_X64_DET_LAYOUT_NCHW) { + uint32_t channel; + for (channel = 0u; channel < op->data.reduce.channels; ++channel) { + const float* source = input + (size_t)channel * op->data.reduce.height * + op->data.reduce.width; + float sum = 0.0f; + uint32_t spatial; + for (spatial = 0u; spatial < op->data.reduce.height * op->data.reduce.width; + ++spatial) { + sum += source[spatial]; + } + output[channel] = sum / (float)(op->data.reduce.height * op->data.reduce.width); + } + } else { +#if defined(LW_WASM_COMPILED_DET) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_reduce_mean_hw_f32(input, output, op->data.reduce.batch, + op->data.reduce.height, op->data.reduce.width, + op->data.reduce.channels); +#else + lw_avx2_nhwc_reduce_mean_hw_f32(input, output, op->data.reduce.batch, + op->data.reduce.height, op->data.reduce.width, + op->data.reduce.channels); +#endif + } + return LW_STATUS_OK; + case LW_X64_DET_OP_AVG_POOL: case LW_X64_DET_OP_MAX_POOL: + input = offset_ptr(instance, op->data.pool.input_offset); + output = offset_ptr(instance, op->data.pool.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.pool.layout == LW_X64_DET_LAYOUT_NCHW) { + if (state != NULL && op->data.pool.input_dimensions[0] == 1 && + op->data.pool.input_dimensions[1] > 1 && + (uint64_t)op->data.pool.input_dimensions[1] * + (uint64_t)op->data.pool.input_dimensions[2] * + (uint64_t)op->data.pool.input_dimensions[3] >= + LW_X64_DET_ELEMENTWISE_SHARD_ELEMS) { + lw_status sharded = run_sharded_elementwise( + instance, op, state, LW_X64_DET_SHARD_POOL_NCHW_CHANNELS, input, NULL, + output, (uint64_t)op->data.pool.input_dimensions[1], 2u, 0); + if (sharded != LW_STATUS_UNSUPPORTED) return sharded; + } + return op->kind == LW_X64_DET_OP_MAX_POOL + ? lw_scalar_max_pool2d_f32(input, output, op->data.pool.input_dimensions, + op->data.pool.output_dimensions, op->data.pool.kernel, + op->data.pool.strides, op->data.pool.pads, 0u) + : lw_scalar_average_pool2d_f32(input, output, op->data.pool.input_dimensions, + op->data.pool.output_dimensions, op->data.pool.kernel, + op->data.pool.strides, op->data.pool.pads, 0u, + op->data.pool.count_include_pad); + } + if (state != NULL && op->data.pool.input_dimensions[0] == 1 && + op->data.pool.strides[0] >= op->data.pool.pads[0] && + op->data.pool.output_dimensions[1] > 1 && + (uint64_t)op->data.pool.input_dimensions[1] * + (uint64_t)op->data.pool.input_dimensions[2] * + (uint64_t)op->data.pool.input_dimensions[3] >= + LW_X64_DET_ELEMENTWISE_SHARD_ELEMS) { + lw_status sharded = run_sharded_elementwise( + instance, op, state, LW_X64_DET_SHARD_POOL_NHWC_ROWS, input, NULL, output, + (uint64_t)op->data.pool.output_dimensions[1], 2u, 0); + if (sharded != LW_STATUS_UNSUPPORTED) return sharded; + } +#if defined(LW_WASM_COMPILED_DET) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_pool_nhwc_f32(input, output, + (uint32_t)op->data.pool.input_dimensions[0], + (uint32_t)op->data.pool.input_dimensions[1], + (uint32_t)op->data.pool.input_dimensions[2], + (uint32_t)op->data.pool.output_dimensions[1], + (uint32_t)op->data.pool.output_dimensions[2], + (uint32_t)op->data.pool.input_dimensions[3], + (uint32_t)op->data.pool.kernel[0], + (uint32_t)op->data.pool.kernel[1], + (uint32_t)op->data.pool.strides[0], + (uint32_t)op->data.pool.strides[1], + (uint32_t)op->data.pool.pads[0], + (uint32_t)op->data.pool.pads[1], + op->data.pool.count_include_pad, op->data.pool.is_max); +#else + lw_avx2_nhwc_pool_f32(input, output, (uint32_t)op->data.pool.input_dimensions[0], + (uint32_t)op->data.pool.input_dimensions[1], + (uint32_t)op->data.pool.input_dimensions[2], + (uint32_t)op->data.pool.output_dimensions[1], + (uint32_t)op->data.pool.output_dimensions[2], + (uint32_t)op->data.pool.input_dimensions[3], + (uint32_t)op->data.pool.kernel[0], (uint32_t)op->data.pool.kernel[1], + (uint32_t)op->data.pool.strides[0], (uint32_t)op->data.pool.strides[1], + (uint32_t)op->data.pool.pads[0], (uint32_t)op->data.pool.pads[1], + op->data.pool.count_include_pad, op->data.pool.is_max); +#endif + return LW_STATUS_OK; + default: return LW_STATUS_UNSUPPORTED; + } +} + +lw_status lw_x64_det_instance_create(const lw_x64_det_program* program, + lw_x64_det_instance** out, lw_error* error) { + lw_x64_det_instance* instance; + if (out == NULL || program == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "DET instance requires a compiled program"); + return LW_STATUS_INVALID_ARGUMENT; + } + *out = NULL; + instance = (lw_x64_det_instance*)calloc(1u, sizeof(*instance)); + if (instance == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "DET instance allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + instance->program = program; + if (program->arena_bytes > 0u) { + instance->arena = (uint8_t*)det_aligned_alloc(64u, (size_t)program->arena_bytes); + } + if (program->scratch_bytes > 0u) { + instance->scratch = (uint8_t*)det_aligned_alloc(64u, (size_t)program->scratch_bytes); + instance->shard_scratch_slice_bytes = program->scratch_bytes; + instance->shard_scratch = (uint8_t*)det_aligned_alloc( + 64u, (size_t)program->scratch_bytes * LW_PARALLEL_MAX_WORKERS); + } + if ((program->arena_bytes > 0u && instance->arena == NULL) || + (program->scratch_bytes > 0u && + (instance->scratch == NULL || instance->shard_scratch == NULL))) { + lw_x64_det_instance_free(instance); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "DET instance workspace allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + *out = instance; + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +void lw_x64_det_instance_free(lw_x64_det_instance* instance) { + if (instance == NULL) return; + det_aligned_free(instance->shard_scratch); + det_aligned_free(instance->scratch); + det_aligned_free(instance->arena); + free(instance); +} + +float* lw_x64_det_instance_input(lw_x64_det_instance* instance, uint64_t* element_count) { + const lw_x64_det_value* value; + if (element_count != NULL) *element_count = 0u; + if (instance == NULL || instance->program == NULL || + instance->program->input_value >= instance->program->value_count) { + return NULL; + } + value = &instance->program->values[instance->program->input_value]; + if (element_count != NULL) *element_count = value->bytes / sizeof(float); + return offset_ptr(instance, value->offset); +} + +static uint32_t det_conv_class(const lw_model* model, uint32_t node_index) { + const uint8_t* node = model->bytes + (size_t)model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + uint64_t param_offset = lwm_read_u64(node + 56u); + const uint8_t* params; + int32_t kernel_height; + int32_t kernel_width; + int32_t stride_height; + int32_t stride_width; + uint32_t group; + if (param_offset == 0u) return LW_EXECUTION_PROFILE_CONV_OTHER; + params = model->bytes + (size_t)param_offset; + kernel_height = lwm_read_i32(params + 8u); + kernel_width = lwm_read_i32(params + 12u); + stride_height = lwm_read_i32(params + 16u); + stride_width = lwm_read_i32(params + 20u); + group = lwm_read_u32(params + 4u); + if (kernel_height == 1 && kernel_width == 1) return LW_EXECUTION_PROFILE_CONV_1X1; + if (kernel_height == 3 && kernel_width == 3 && group > 1u) { + return LW_EXECUTION_PROFILE_CONV_DEPTHWISE_3X3; + } + if (kernel_height == 3 && kernel_width == 3 && stride_height == 2 && stride_width == 2) { + return LW_EXECUTION_PROFILE_CONV_STRIDE2_3X3; + } + if (kernel_height == 3 && kernel_width == 3) return LW_EXECUTION_PROFILE_CONV_3X3; + return LW_EXECUTION_PROFILE_CONV_OTHER; +} + +/* Attribute one physical op's elapsed time to its semantic span, mirroring + * the canonical executor's accounting (fused GELU work lands on the Erf node + * with 1-ns placeholders for the other four). */ +static void profile_physical_op(const lw_model* model, lw_execution_profile* profile, + const lw_x64_det_op* op, uint64_t elapsed) { + uint32_t offset; + uint32_t begin = op->semantic_begin; + uint32_t count = op->semantic_count; + if (profile == NULL || count == 0u) return; + if (count == 5u && begin + 5u <= model->info.node_count) { + for (offset = 0u; offset < 5u; ++offset) { + uint32_t node_index = begin + offset; + const uint8_t* node = model->bytes + (size_t)model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + uint32_t operation = (uint32_t)lwm_read_u16(node); + uint64_t node_elapsed = offset == 1u ? (elapsed == 0u ? 1u : elapsed) : 1u; + if (operation < LW_EXECUTION_PROFILE_OPERATOR_CAPACITY && + profile->operator_nanoseconds[operation] <= UINT64_MAX - node_elapsed && + profile->operator_invocations[operation] != UINT64_MAX) { + profile->operator_nanoseconds[operation] += node_elapsed; + profile->operator_invocations[operation] += 1u; + } + if (node_index < LW_EXECUTION_PROFILE_NODE_CAPACITY && + profile->node_nanoseconds[node_index] <= UINT64_MAX - node_elapsed && + profile->node_invocations[node_index] != UINT64_MAX) { + profile->node_nanoseconds[node_index] += node_elapsed; + profile->node_invocations[node_index] += 1u; + } + } + return; + } + for (offset = 0u; offset < count && begin + offset < model->info.node_count; ++offset) { + uint32_t node_index = begin + offset; + const uint8_t* node = model->bytes + (size_t)model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + uint32_t operation = (uint32_t)lwm_read_u16(node); + if (operation < LW_EXECUTION_PROFILE_OPERATOR_CAPACITY && + profile->operator_nanoseconds[operation] <= UINT64_MAX - elapsed && + profile->operator_invocations[operation] != UINT64_MAX) { + profile->operator_nanoseconds[operation] += elapsed; + profile->operator_invocations[operation] += 1u; + } + if (node_index < LW_EXECUTION_PROFILE_NODE_CAPACITY && + profile->node_nanoseconds[node_index] <= UINT64_MAX - elapsed && + profile->node_invocations[node_index] != UINT64_MAX) { + profile->node_nanoseconds[node_index] += elapsed; + profile->node_invocations[node_index] += 1u; + } + if (operation == LW_OP_CONV && node_index < LW_EXECUTION_PROFILE_NODE_CAPACITY) { + uint32_t conv_class = det_conv_class(model, node_index); + if (profile->conv_class_nanoseconds[conv_class] <= UINT64_MAX - elapsed && + profile->conv_class_invocations[conv_class] != UINT64_MAX) { + profile->conv_class_nanoseconds[conv_class] += elapsed; + profile->conv_class_invocations[conv_class] += 1u; + } + } + } +} + +static lw_status run_backbone_ops(lw_x64_det_instance* instance, + const lw_x64_det_run_state* state, lw_error* error) { + uint32_t i; + lw_status status; + if (instance == NULL || instance->program == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "DET instance is invalid"); + return LW_STATUS_INVALID_ARGUMENT; + } + for (i = 0u; i < instance->program->op_count; ++i) { + status = execute_op(instance, &instance->program->ops[i], state, error); + if (status != LW_STATUS_OK) { + if (error != NULL && error->message[0] == '\0') { + char message[96]; + (void)snprintf(message, sizeof(message), "DET physical op %u kind %u failed", + (unsigned)i, (unsigned)instance->program->ops[i].kind); + lw_set_error(error, status, message); + } + return status; + } + } + return LW_STATUS_OK; +} + +lw_status lw_x64_det_instance_run_op(lw_x64_det_instance* instance, uint32_t op_index, + lw_error* error) { + if (instance == NULL || instance->program == NULL || + op_index >= instance->program->op_count) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "DET physical op index is invalid"); + return LW_STATUS_INVALID_ARGUMENT; + } + return execute_op(instance, &instance->program->ops[op_index], NULL, error); +} + +lw_status lw_x64_det_instance_run_op_ex(lw_x64_det_instance* instance, uint32_t op_index, + lw_thread_pool* pool, uint32_t worker_count, + lw_error* error) { + lw_x64_det_run_state state; + if (instance == NULL || instance->program == NULL || + op_index >= instance->program->op_count) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "DET physical op index is invalid"); + return LW_STATUS_INVALID_ARGUMENT; + } + state.instance = instance; + state.pool = pool; + state.worker_count = worker_count; + return execute_op(instance, &instance->program->ops[op_index], &state, error); +} + +lw_status lw_x64_det_instance_run(lw_x64_det_instance* instance, lw_error* error) { + lw_status status = run_backbone_ops(instance, NULL, error); + if (status == LW_STATUS_OK) lw_set_error(error, LW_STATUS_OK, ""); + return status; +} + +lw_status lw_x64_det_instance_run_profiled(lw_x64_det_instance* instance, float* output, + uint64_t output_element_count, + lw_execution_profile* profile, lw_error* error) { + return lw_x64_det_instance_run_profiled_ex(instance, output, output_element_count, NULL, 1u, + profile, error); +} + +/* Workers actually used by a physical op, recomputed exactly like the + * dispatch so the thread histograms match the executed shards. */ +static uint32_t det_op_workers(const lw_x64_det_run_state* state, const lw_x64_det_op* op) { + switch (op->kind) { + case LW_X64_DET_OP_POINTWISE: + case LW_X64_DET_OP_DENSE: + return det_shard_workers(state, conv_macs(&op->data.conv), + LW_X64_DET_CONV_SHARD_MACS); + case LW_X64_DET_OP_DEPTHWISE: + return det_shard_workers(state, conv_macs(&op->data.conv), + LW_X64_DET_DEPTHWISE_SHARD_MACS); + case LW_X64_DET_OP_CONV_TRANSPOSE_NHWC: + return det_shard_workers( + state, (uint64_t)op->data.conv_transpose.input_dimensions[1] * + op->data.conv_transpose.output_dimensions[1] * 4u * + op->data.conv_transpose.output_dimensions[2] * + op->data.conv_transpose.output_dimensions[3], + LW_X64_DET_CONV_SHARD_MACS); + case LW_X64_DET_OP_CONV_TRANSPOSE_NHWC_C1: + return det_shard_workers( + state, (uint64_t)op->data.conv_transpose.input_dimensions[1] * + op->data.conv_transpose.output_dimensions[1] * 4u * + op->data.conv_transpose.output_dimensions[2] * + op->data.conv_transpose.output_dimensions[3], + LW_X64_DET_DEPTHWISE_SHARD_MACS); + default: + return 1u; + } +} + +lw_status lw_x64_det_instance_run_profiled_ex(lw_x64_det_instance* instance, float* output, + uint64_t output_element_count, + lw_thread_pool* pool, uint32_t worker_count, + lw_execution_profile* profile, lw_error* error) { + const lw_x64_det_program* program; + const lw_x64_det_value* output_value; + lw_x64_det_run_state state; + uint64_t total_ns = 0u; + lw_status status; + uint32_t i; + if (instance == NULL || instance->program == NULL || output == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "DET instance, program and output are required"); + return LW_STATUS_INVALID_ARGUMENT; + } + program = instance->program; + if (program->output_value >= program->value_count) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "DET program has no graph output"); + return LW_STATUS_INVALID_SHAPE; + } + output_value = &program->values[program->output_value]; + if (output_element_count < output_value->bytes / sizeof(float)) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "DET output buffer is too small"); + return LW_STATUS_OUT_OF_BOUNDS; + } + state.instance = instance; + state.pool = pool; + state.worker_count = worker_count; + memset(&instance->profile, 0, sizeof(instance->profile)); + if (profile != NULL) { + profile->layout_candidate_nodes += program->nhwc_effective_nodes + + program->nchw_effective_nodes; + profile->layout_selected_nodes += program->nhwc_effective_nodes; + profile->layout_fallback_nodes += program->nchw_effective_nodes; + profile->layout_analysis_runs += 1u; + } + for (i = 0u; i < program->op_count; ++i) { + const lw_x64_det_op* op = &program->ops[i]; + uint64_t started = profile != NULL && profile->clock != NULL + ? profile->clock(profile->clock_context) : 0u; + status = execute_op(instance, op, &state, error); + if (status != LW_STATUS_OK) { + if (error != NULL && error->message[0] == '\0') { + char message[96]; + (void)snprintf(message, sizeof(message), "DET physical op %u kind %u failed", + (unsigned)i, (unsigned)op->kind); + lw_set_error(error, status, message); + } + return status; + } + if (profile != NULL && profile->clock != NULL) { + uint64_t finished = profile->clock(profile->clock_context); + uint64_t elapsed = finished >= started ? finished - started : 0u; + uint32_t op_workers = det_op_workers(&state, op); + total_ns += elapsed; + switch (op->kind) { + case LW_X64_DET_OP_POINTWISE: instance->profile.pointwise_ns += elapsed; break; + case LW_X64_DET_OP_DENSE: instance->profile.dense_ns += elapsed; break; + case LW_X64_DET_OP_DEPTHWISE: instance->profile.depthwise_ns += elapsed; break; + case LW_X64_DET_OP_CONV_NCHW: instance->profile.conv_nchw_ns += elapsed; break; + case LW_X64_DET_OP_ADD: + case LW_X64_DET_OP_MUL: + case LW_X64_DET_OP_DIV: instance->profile.binary_ns += elapsed; break; + case LW_X64_DET_OP_RELU: + case LW_X64_DET_OP_ERF: + case LW_X64_DET_OP_GELU: + case LW_X64_DET_OP_HARD_SIGMOID: instance->profile.unary_ns += elapsed; break; + case LW_X64_DET_OP_REDUCE_MEAN: instance->profile.reduce_ns += elapsed; break; + case LW_X64_DET_OP_AVG_POOL: + case LW_X64_DET_OP_MAX_POOL: instance->profile.pool_ns += elapsed; break; + case LW_X64_DET_OP_LAYOUT_CONVERT: + instance->profile.convert_ns += elapsed; + profile->layout_transform_nanoseconds += elapsed; + profile->layout_transform_invocations += 1u; + profile->layout_transform_bytes += + (uint64_t)op->data.convert.batch * op->data.convert.channels * + op->data.convert.height * op->data.convert.width * sizeof(float); + break; + case LW_X64_DET_OP_CONV_TRANSPOSE_NCHW: + case LW_X64_DET_OP_CONV_TRANSPOSE_NHWC: + case LW_X64_DET_OP_CONV_TRANSPOSE_NHWC_C1: + instance->profile.conv_transpose_ns += elapsed; + break; + case LW_X64_DET_OP_RESIZE_NCHW: + case LW_X64_DET_OP_RESIZE_NHWC: + instance->profile.resize_ns += elapsed; + break; + case LW_X64_DET_OP_SIGMOID_NCHW: instance->profile.sigmoid_ns += elapsed; break; + case LW_X64_DET_OP_CONCAT_NCHW: + case LW_X64_DET_OP_CONCAT_NHWC: + instance->profile.concat_ns += elapsed; + break; + default: break; + } + if (op_workers >= 1u && op_workers < LW_EXECUTION_PROFILE_THREAD_HISTOGRAM_CAPACITY) { + if (op->kind == LW_X64_DET_OP_CONV_TRANSPOSE_NHWC || + op->kind == LW_X64_DET_OP_CONV_TRANSPOSE_NHWC_C1) { + profile->conv_transpose_thread_histogram[op_workers] += 1u; + } else if (op->kind == LW_X64_DET_OP_POINTWISE || + op->kind == LW_X64_DET_OP_DENSE || + op->kind == LW_X64_DET_OP_DEPTHWISE || + op->kind == LW_X64_DET_OP_CONV_NCHW) { + profile->conv_thread_histogram[op_workers] += 1u; + } + } + profile_physical_op(program->model, profile, op, elapsed); + } + } + instance->profile.total_ns = total_ns; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_COMPILED_DET) + if (profile != NULL && profile->clock != NULL && + lw_profile_env_present("LW_WASM_OCR_PROFILE")) { + const lw_x64_det_profile* det_profile = &instance->profile; + const double to_ms = 1.0 / 1000000.0; + (void)fprintf(stderr, + "LW_WASM_DET_PROFILE total=%.3f pointwise=%.3f dense=%.3f " + "depthwise=%.3f convtranspose=%.3f binary=%.3f pool=%.3f " + "concat=%.3f resize=%.3f other=%.3f\n", + det_profile->total_ns * to_ms, + det_profile->pointwise_ns * to_ms, + det_profile->dense_ns * to_ms, + det_profile->depthwise_ns * to_ms, + det_profile->conv_transpose_ns * to_ms, + det_profile->binary_ns * to_ms, + det_profile->pool_ns * to_ms, + det_profile->concat_ns * to_ms, + det_profile->resize_ns * to_ms, + (det_profile->total_ns - det_profile->pointwise_ns - + det_profile->dense_ns - det_profile->depthwise_ns - + det_profile->conv_transpose_ns - det_profile->binary_ns - + det_profile->pool_ns - det_profile->concat_ns - + det_profile->resize_ns) * to_ms); + } +#endif + if (output_value->layout == LW_X64_DET_LAYOUT_NCHW || output_value->alt_producer < 0) { + memcpy(output, offset_ptr(instance, output_value->offset), + (size_t)(output_value->bytes)); + } else { + memcpy(output, offset_ptr(instance, output_value->alt_offset), + (size_t)(output_value->bytes)); + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_det_backend_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_det_backend_internal.h new file mode 100644 index 00000000..630ac057 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_det_backend_internal.h @@ -0,0 +1,409 @@ +#ifndef LW_X64_DET_BACKEND_INTERNAL_H +#define LW_X64_DET_BACKEND_INTERNAL_H + +/* + * Standalone x64 DET graph backend (experimental, AVX2+FMA only). Cloned from + * the proven x64_rec_backend structure: two-pass lowering into a physical op + * table, a greedy-by-size arena interval planner, and a per-instance arena. + * The one new design piece over the REC clone is the effective-layout walk: + * the planner's NHWC classification is crossed with a v1 lowerability set + * (Resize/ConvTranspose/Concat/Sigmoid stay NCHW for now), and tensors that + * cross an effective-layout boundary get a lazily materialized LAYOUT_CONVERT + * physical op writing into a dedicated alternate arena slot. + */ + +#include "cpu_features.h" +#include "model_internal.h" +#include "session_internal.h" +#include "executor_internal.h" +#include "../kernels/nhwc_internal.h" +#include "../kernels/scalar_kernels.h" + +#include +#include + +/* Provided by x64_rec_backend_arena.c (compiled under the same gate). */ +uint64_t lw_x64_rec_arena_align(uint64_t value, uint64_t alignment); +lw_status lw_x64_rec_arena_alloc(uint64_t* cursor, uint64_t bytes, uint64_t alignment, + uint64_t* out_offset, lw_error* error); + +/* Provided by layout_convert_x64.c (same gate; declared again here so the DET + * backend does not depend on the REC fast-plan header). */ +void lw_x64_fast_nchw_to_nhwc(const float* source, float* destination, + uint32_t batch, uint32_t channels, + uint32_t height, uint32_t width); +void lw_x64_fast_nhwc_to_nchw(const float* source, float* destination, + uint32_t batch, uint32_t channels, + uint32_t height, uint32_t width); + +typedef enum lw_x64_det_compile_result { + LW_X64_DET_COMPILE_OK = 0, + LW_X64_DET_COMPILE_UNSUPPORTED = 1, + LW_X64_DET_COMPILE_INVALID_GRAPH = 2, + LW_X64_DET_COMPILE_OUT_OF_MEMORY = 3 +} lw_x64_det_compile_result; + +typedef enum lw_x64_det_compile_strategy { + LW_X64_DET_COMPILE_NCHW = 0, + LW_X64_DET_COMPILE_NHWC = 1 +} lw_x64_det_compile_strategy; + +typedef enum lw_x64_det_storage_layout { + LW_X64_DET_LAYOUT_NCHW = 0, + LW_X64_DET_LAYOUT_NHWC = 1 +} lw_x64_det_storage_layout; + +typedef enum lw_x64_det_op_kind { + LW_X64_DET_OP_DENSE = 1, + LW_X64_DET_OP_POINTWISE = 2, + LW_X64_DET_OP_DEPTHWISE = 3, + LW_X64_DET_OP_AFFINE = 4, + LW_X64_DET_OP_ADD = 5, + LW_X64_DET_OP_MUL = 6, + LW_X64_DET_OP_DIV = 7, + LW_X64_DET_OP_RELU = 8, + LW_X64_DET_OP_ERF = 9, + LW_X64_DET_OP_GELU = 10, + LW_X64_DET_OP_HARD_SIGMOID = 11, + LW_X64_DET_OP_REDUCE_MEAN = 12, + LW_X64_DET_OP_AVG_POOL = 13, + LW_X64_DET_OP_MAX_POOL = 14, + LW_X64_DET_OP_CONV_NCHW = 15, + LW_X64_DET_OP_LAYOUT_CONVERT = 16, + LW_X64_DET_OP_CONV_TRANSPOSE_NCHW = 17, + LW_X64_DET_OP_RESIZE_NCHW = 18, + LW_X64_DET_OP_SIGMOID_NCHW = 19, + LW_X64_DET_OP_CONCAT_NCHW = 20, + LW_X64_DET_OP_CONV_TRANSPOSE_NHWC = 21, + LW_X64_DET_OP_CONV_TRANSPOSE_NHWC_C1 = 22, + LW_X64_DET_OP_RESIZE_NHWC = 23, + LW_X64_DET_OP_CONCAT_NHWC = 24, + LW_X64_DET_OP_GENERIC_UNSUPPORTED = 0xffff +} lw_x64_det_op_kind; + +typedef enum lw_x64_det_conv_kind { + LW_X64_DET_CONV_POINTWISE = 1, + LW_X64_DET_CONV_DEPTHWISE = 2, + LW_X64_DET_CONV_DENSE = 3 +} lw_x64_det_conv_kind; + +typedef enum lw_x64_det_pointwise_kernel { + LW_X64_DET_PW_6X16 = 0, + LW_X64_DET_PW_4X16 = 1, + LW_X64_DET_PW_3X32 = 2, + LW_X64_DET_PW_2X32 = 3 +} lw_x64_det_pointwise_kernel; + +typedef enum lw_x64_det_nchw_pointwise_kernel { + LW_X64_DET_NCHW_PW_FMA4 = 0, + LW_X64_DET_NCHW_PW_FMA8 = 1 +} lw_x64_det_nchw_pointwise_kernel; + +typedef enum lw_x64_det_nchw_conv_kernel { + LW_X64_DET_NCHW_CONV_POINTWISE_PACKED = 0, + LW_X64_DET_NCHW_CONV_3X3S2 = 1, + LW_X64_DET_NCHW_CONV_3X3_UNIT = 2, + LW_X64_DET_NCHW_CONV_2X2_PADEND1 = 3, + LW_X64_DET_NCHW_CONV_DEPTHWISE3X3 = 4, + LW_X64_DET_NCHW_CONV_DEPTHWISE3X3_S2X1 = 5, + LW_X64_DET_NCHW_CONV_DEPTHWISE5X5 = 6, + LW_X64_DET_NCHW_CONV_SCALAR = 7 +} lw_x64_det_nchw_conv_kernel; + +typedef enum lw_x64_det_broadcast_kind { + LW_X64_DET_BROADCAST_SAME = 0, + LW_X64_DET_BROADCAST_RIGHT_SCALAR = 1, + LW_X64_DET_BROADCAST_LEFT_SCALAR = 2, + LW_X64_DET_BROADCAST_RIGHT_CHANNEL = 3, + LW_X64_DET_BROADCAST_LEFT_CHANNEL = 4, + LW_X64_DET_BROADCAST_GENERAL = 5 +} lw_x64_det_broadcast_kind; + +typedef struct lw_x64_det_value { + uint64_t offset; + /* Alternate-layout copy written by a LAYOUT_CONVERT op. Zero until the + * arena plan assigns it; alt_producer == -1 means "not needed". */ + uint64_t alt_offset; + uint64_t bytes; + const float* constant_data; + int32_t producer; + int32_t last_use; + int32_t alt_producer; + int32_t alt_last_use; + int32_t dimensions[4]; + uint32_t rank; + uint8_t layout; + uint8_t alt_layout; + uint8_t alias; + uint8_t reserved; + uint32_t alias_of; +} lw_x64_det_value; + +typedef struct lw_x64_det_conv_op { + uint64_t input_offset; + uint64_t output_offset; + const float* packed_weights; + const float* original_weights; + const float* bias; + /* Folded epilogue extras: a per-channel constant Add applied to the + * finished accumulator (post_bias), and an elementwise residual Add read + * from the arena at residual_offset when has_residual != 0. */ + const float* post_bias; + uint64_t residual_offset; + uint8_t has_residual; + uint8_t scalar_fallback; + uint8_t pointwise_kernel; + uint8_t nchw_pointwise_kernel; + uint8_t nchw_conv_kernel; + uint32_t input_channels; + uint32_t output_channels; + uint32_t input_height; + uint32_t input_width; + uint32_t output_height; + uint32_t output_width; + uint32_t kernel_h; + uint32_t kernel_w; + uint32_t weight_h; + uint32_t weight_w; + uint32_t stride_h; + uint32_t stride_w; + uint32_t pad_top; + uint32_t pad_left; + uint32_t pad_bottom; + uint32_t pad_right; + uint32_t groups; + uint32_t dense_kc; + uint64_t scratch_bytes; + uint16_t kernel_kind; + uint16_t activation; +} lw_x64_det_conv_op; + +typedef struct lw_x64_det_binary_op { + uint64_t left_offset; + uint64_t right_offset; + uint64_t output_offset; + const float* left_constant; + const float* right_constant; + uint64_t element_count; + uint32_t pixels; + uint32_t channels; + uint16_t operation; + uint8_t broadcast_kind; + uint8_t layout; + int32_t dimensions[4]; +} lw_x64_det_binary_op; + +typedef struct lw_x64_det_unary_op { + uint64_t input_offset; + uint64_t output_offset; + uint64_t element_count; + float alpha; + float beta; + int32_t dimensions[4]; + uint32_t rank; + uint8_t layout; + uint8_t reserved[3]; +} lw_x64_det_unary_op; + +typedef struct lw_x64_det_affine_op { + uint64_t input_offset; + uint64_t output_offset; + const float* mul; + const float* add; + const float* scale; + const float* bias; + const float* mean; + const float* variance; + float epsilon; + uint8_t channel_major; + uint32_t pixels; + uint32_t channels; +} lw_x64_det_affine_op; + +typedef struct lw_x64_det_reduce_op { + uint64_t input_offset; + uint64_t output_offset; + uint32_t batch; + uint32_t height; + uint32_t width; + uint32_t channels; + uint8_t layout; + uint8_t reserved[3]; +} lw_x64_det_reduce_op; + +typedef struct lw_x64_det_pool_op { + uint64_t input_offset; + uint64_t output_offset; + int32_t input_dimensions[4]; + int32_t output_dimensions[4]; + int32_t kernel[2]; + int32_t strides[2]; + int32_t pads[4]; + uint8_t count_include_pad; + uint8_t is_max; + uint8_t layout; + uint8_t reserved; +} lw_x64_det_pool_op; + +typedef struct lw_x64_det_convert_op { + uint64_t input_offset; + uint64_t output_offset; + uint32_t batch; + uint32_t channels; + uint32_t height; + uint32_t width; + uint8_t to_nhwc; + uint8_t reserved[3]; +} lw_x64_det_convert_op; + +typedef struct lw_x64_det_conv_transpose_op { + uint64_t input_offset; + uint64_t output_offset; + const float* weights; + const float* packed_weights; + const float* bias; + int32_t input_dimensions[4]; + int32_t output_dimensions[4]; + uint8_t layout; + uint8_t reserved[3]; +} lw_x64_det_conv_transpose_op; + +typedef struct lw_x64_det_resize_op { + uint64_t input_offset; + uint64_t output_offset; + uint32_t rank; + int32_t input_dimensions[4]; + int32_t output_dimensions[4]; + float scales[4]; + uint8_t layout; + uint8_t reserved[3]; +} lw_x64_det_resize_op; + +typedef struct lw_x64_det_concat_op { + uint64_t input_offsets[LWM_V0_MAX_NODE_INPUTS]; + uint64_t output_offset; + uint32_t input_count; + uint32_t input_ranks[LWM_V0_MAX_NODE_INPUTS]; + int32_t input_dimensions[LWM_V0_MAX_NODE_INPUTS][4]; + uint32_t output_rank; + int32_t output_dimensions[4]; + int32_t axis; + uint8_t layout; + uint8_t reserved[3]; +} lw_x64_det_concat_op; + +typedef struct lw_x64_det_op { + uint16_t kind; + uint16_t reserved; + uint32_t semantic_begin; + uint16_t semantic_count; + uint16_t flags; + union { + lw_x64_det_conv_op conv; + lw_x64_det_binary_op binary; + lw_x64_det_unary_op unary; + lw_x64_det_affine_op affine; + lw_x64_det_reduce_op reduce; + lw_x64_det_pool_op pool; + lw_x64_det_convert_op convert; + lw_x64_det_conv_transpose_op conv_transpose; + lw_x64_det_resize_op resize; + lw_x64_det_concat_op concat; + } data; +} lw_x64_det_op; + +typedef struct lw_x64_det_constant { + void* data; + uint64_t bytes; +} lw_x64_det_constant; + +typedef struct lw_x64_det_program { + const lw_model* model; + lw_cpu_capabilities cpu; + uint64_t model_signature; + uint32_t input_height; + uint32_t input_width; + uint32_t input_value; + uint32_t output_value; + lw_x64_det_op* ops; + uint32_t op_count; + lw_x64_det_value* values; + uint32_t value_count; + uint32_t semantic_consumed; + uint32_t semantic_elided; + uint32_t semantic_fused; + uint32_t unsupported_nodes; + uint32_t layout_conversions; + uint8_t direct_nhwc; + uint8_t backend_layout; + uint16_t reserved; + uint64_t arena_bytes; + uint64_t scratch_bytes; + uint32_t packed_constant_count; + lw_x64_det_constant* constants; + /* Effective-layout walk output, kept for the contract driver. */ + uint8_t* effective_node_layout; + uint32_t nhwc_effective_nodes; + uint32_t nchw_effective_nodes; +} lw_x64_det_program; + +typedef struct lw_x64_det_profile { + uint64_t total_ns; + uint64_t pointwise_ns; + uint64_t dense_ns; + uint64_t depthwise_ns; + uint64_t conv_nchw_ns; + uint64_t binary_ns; + uint64_t unary_ns; + uint64_t reduce_ns; + uint64_t pool_ns; + uint64_t convert_ns; + uint64_t conv_transpose_ns; + uint64_t resize_ns; + uint64_t sigmoid_ns; + uint64_t concat_ns; +} lw_x64_det_profile; + +typedef struct lw_x64_det_instance { + const lw_x64_det_program* program; + uint8_t* arena; + uint8_t* scratch; + /* Per-worker dense-kernel scratch slices (64B-aligned, one per pool + * worker) so sharded dense ops never share a patch buffer. */ + uint8_t* shard_scratch; + uint64_t shard_scratch_slice_bytes; + lw_x64_det_profile profile; +} lw_x64_det_instance; + +lw_x64_det_compile_result lw_x64_det_backend_compile_ex( + const lw_model* model, uint32_t height, uint32_t width, + lw_x64_det_compile_strategy strategy, lw_x64_det_program** out_program, + lw_error* error); +void lw_x64_det_program_free(lw_x64_det_program* program); +lw_status lw_x64_det_instance_create(const lw_x64_det_program* program, + lw_x64_det_instance** out, lw_error* error); +void lw_x64_det_instance_free(lw_x64_det_instance* instance); +float* lw_x64_det_instance_input(lw_x64_det_instance* instance, uint64_t* element_count); +lw_status lw_x64_det_instance_run(lw_x64_det_instance* instance, lw_error* error); +/* Timed run that maps physical-op elapsed time onto the semantic profile and + * copies the graph-output value (always NCHW [1,1,H,W]) into `output`. */ +lw_status lw_x64_det_instance_run_profiled(lw_x64_det_instance* instance, float* output, + uint64_t output_element_count, + lw_execution_profile* profile, lw_error* error); +/* Threaded variant: shardable physical ops fan out over `pool` (worker 0 runs + * on the caller thread) up to `worker_count` workers. Whole-tile/whole-row + * partitioning keeps sharded output bit-identical to the serial run. */ +lw_status lw_x64_det_instance_run_profiled_ex(lw_x64_det_instance* instance, float* output, + uint64_t output_element_count, + lw_thread_pool* pool, uint32_t worker_count, + lw_execution_profile* profile, lw_error* error); +/* Test-only internal hook: execute one compiled physical op. */ +lw_status lw_x64_det_instance_run_op(lw_x64_det_instance* instance, uint32_t op_index, + lw_error* error); +/* Test-only internal hook: execute one physical op through the threaded + * sharding dispatch (pool NULL keeps the serial path). */ +lw_status lw_x64_det_instance_run_op_ex(lw_x64_det_instance* instance, uint32_t op_index, + lw_thread_pool* pool, uint32_t worker_count, + lw_error* error); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_arena.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_arena.c new file mode 100644 index 00000000..533e6fff --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_arena.c @@ -0,0 +1,27 @@ +#include "x64_rec_backend_internal.h" +#include + +uint64_t lw_x64_rec_arena_align(uint64_t value, uint64_t alignment) { + uint64_t mask; + if (alignment == 0u || (alignment & (alignment - 1u)) != 0u) return UINT64_MAX; + mask = alignment - 1u; + if (value > UINT64_MAX - mask) return UINT64_MAX; + return (value + mask) & ~mask; +} + +lw_status lw_x64_rec_arena_alloc(uint64_t* cursor, uint64_t bytes, uint64_t alignment, + uint64_t* out_offset, lw_error* error) { + uint64_t start; + if (cursor == NULL || out_offset == NULL || alignment == 0u) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid REC arena allocation"); + return LW_STATUS_INVALID_ARGUMENT; + } + start = lw_x64_rec_arena_align(*cursor, alignment); + if (start == UINT64_MAX || bytes > UINT64_MAX - start) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "REC arena size overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + *out_offset = start; + *cursor = start + bytes; + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_compile.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_compile.c new file mode 100644 index 00000000..b5ef60d9 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_compile.c @@ -0,0 +1,1294 @@ +#include "x64_rec_backend_internal.h" +#include "ctc_projection_internal.h" +#include "lwm_read.h" +#include "operator_internal.h" +#include "packed_matmul_internal.h" + +#include +#include +#include +#include +#include +#include + +static int rec_backend_cpu_supported(lw_cpu_capabilities cpu) { +#if defined(__EMSCRIPTEN__) && defined(__wasm_simd128__) + (void)cpu; + return 1; +#else + return lw_simd_level_is_avx2(cpu.simd) && cpu.has_avx2_fma; +#endif +} + +static lw_status make_shape_session(const lw_model* model, uint32_t height, uint32_t width, + lw_session** out, lw_error* error) { + lw_tensor_desc input; + lw_tensor_desc_init(&input); + input.dtype = LW_DTYPE_F32; + input.rank = 4u; + input.dimensions[0] = 1; + input.dimensions[1] = 3; + input.dimensions[2] = (int32_t)height; + input.dimensions[3] = (int32_t)width; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_LAZY_FALLBACK) + /* The physical compiler consumes resolved tensor metadata, never the + * canonical execution workspace. Avoid growing the WASM heap to the + * canonical workspace high-water mark during each width compile. */ + return lw_session_create_metadata_only(model, &input, 1u, NULL, out, error); +#else + return lw_session_create(model, &input, 1u, NULL, out, error); +#endif +} + +static const uint8_t* node_bytes(const lw_model* model, uint32_t index) { + return model->bytes + (size_t)model->node_offset + (size_t)index * LWM_V0_NODE_SIZE; +} + +static const float* constant_f32(const lw_model* model, uint32_t index) { + const uint8_t* tensor = model->bytes + (size_t)model->tensor_offset + + (size_t)index * LWM_V0_TENSOR_SIZE; + return (const float*)(const void*)(model->bytes + (size_t)lwm_read_u64(tensor + 48u)); +} + +static int rec_value_is_nhwc_physical(const lw_model* model, const lw_session* session, + uint32_t tensor); + +/* Physical extent of the fastest-varying (channel) axis for a broadcast + * against this output value: NHWC-stored rank-4 values keep C last, + * canonical-order rank-4 values (attention chain) keep W last, and + * lower ranks use the canonical trailing axis directly. */ +static uint32_t rec_binary_channel_extent(const lw_model* model, const lw_session* session, + const lw_x64_rec_value* out_value, + uint32_t out_index) { + if (out_value->rank == 0u) return 1u; + if (out_value->rank == 4u && + !rec_value_is_nhwc_physical(model, session, out_index)) { + return (uint32_t)out_value->dimensions[2]; + } + return (uint32_t)out_value->dimensions[out_value->rank - 1u]; +} +static float read_f32(const uint8_t* p) { + uint32_t bits = lwm_read_u32(p); + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static void fill_value(const lw_session* session, uint32_t i, lw_x64_rec_value* value, + lw_x64_rec_compile_strategy strategy) { + const lw_runtime_tensor* tensor = &session->tensors[i]; + uint32_t j; + memset(value, 0, sizeof(*value)); + value->bytes = tensor->byte_size; + value->rank = tensor->rank > 4u ? 4u : tensor->rank; + if (tensor->rank == 4u) { + if (strategy == LW_X64_REC_COMPILE_NCHW) { + value->dimensions[0] = tensor->dimensions[0]; + value->dimensions[1] = tensor->dimensions[1]; + value->dimensions[2] = tensor->dimensions[2]; + value->dimensions[3] = tensor->dimensions[3]; + value->layout = LW_X64_REC_LAYOUT_NCHW; + } else { + value->dimensions[0] = tensor->dimensions[0]; + value->dimensions[1] = tensor->dimensions[2]; + value->dimensions[2] = tensor->dimensions[3]; + value->dimensions[3] = tensor->dimensions[1]; + value->layout = LW_X64_REC_LAYOUT_NHWC; + } + } else { + for (j = 0u; j < value->rank; ++j) value->dimensions[j] = tensor->dimensions[j]; + value->layout = LW_X64_REC_LAYOUT_SCALAR; + } + value->producer = -1; + value->last_use = -1; + if ((tensor->flags & LWM_V0_TENSOR_FLAG_CONSTANT) != 0u) value->constant_data = constant_f32(session->model, i); +} + +static int add_constant(lw_x64_rec_program* program, void* data, uint64_t bytes) { + lw_x64_rec_constant* next; + if (data == NULL || bytes == 0u) return 0; + next = (lw_x64_rec_constant*)realloc(program->constants, + (size_t)(program->packed_constant_count + 1u) * sizeof(*next)); + if (next == NULL) return 0; + program->constants = next; + program->constants[program->packed_constant_count].data = data; + program->constants[program->packed_constant_count].bytes = bytes; + ++program->packed_constant_count; + program->owned_constant_bytes += bytes; + return 1; +} + +static void* alloc_constant(lw_x64_rec_program* program, uint64_t bytes) { + void* data; + if (bytes == 0u || bytes > SIZE_MAX) return NULL; + data = malloc((size_t)bytes); + if (data == NULL || !add_constant(program, data, bytes)) { + free(data); + return NULL; + } + return data; +} + +/* A width-specific op can borrow only a pack with the same semantic node and + * packing contract. The root stays alive through packed_source's reference. */ +static const lw_x64_rec_op* find_packed_source_op(const lw_x64_rec_program* program, + uint32_t semantic_begin) { + const lw_x64_rec_program* source = program->packed_source; + uint32_t i; + if (source == NULL) return NULL; + for (i = 0u; i < source->op_count; ++i) { + if (source->ops[i].semantic_begin == semantic_begin) return &source->ops[i]; + } + return NULL; +} + +static void note_borrow(lw_x64_rec_program* program, uint64_t bytes) { + program->borrowed_constant_bytes += bytes; + ++program->borrowed_constant_count; +} + +static const float* borrow_conv_pack(lw_x64_rec_program* program, + const lw_x64_rec_op* op, uint64_t bytes) { + const lw_x64_rec_op* source = find_packed_source_op(program, op->semantic_begin); + if (source == NULL || source->kind != op->kind || + source->data.conv.packed_weights == NULL || + source->data.conv.kernel_kind != op->data.conv.kernel_kind || + source->data.conv.input_channels != op->data.conv.input_channels || + source->data.conv.output_channels != op->data.conv.output_channels || + source->data.conv.kernel_h != op->data.conv.kernel_h || + source->data.conv.kernel_w != op->data.conv.kernel_w || + source->data.conv.original_weights != op->data.conv.original_weights) + return NULL; + note_borrow(program, bytes); + return source->data.conv.packed_weights; +} + +static const float* borrow_matmul_pack(lw_x64_rec_program* program, + const lw_x64_rec_op* op, uint64_t bytes) { + const lw_x64_rec_op* source = find_packed_source_op(program, op->semantic_begin); + if (source == NULL || source->kind != LW_X64_REC_OP_MATMUL || + source->data.matmul.packed_weights == NULL || + source->data.matmul.inner != op->data.matmul.inner || + source->data.matmul.columns != op->data.matmul.columns || + source->data.matmul.weights != op->data.matmul.weights) + return NULL; + note_borrow(program, bytes); + return source->data.matmul.packed_weights; +} + +static int detect_ctc(const lw_model* model, const lw_session* session, lw_x64_rec_ctc_tail* tail) { + uint32_t n = model->info.node_count; + const uint8_t* matmul; + const uint8_t* add; + const uint8_t* softmax; + uint32_t activation; + uint32_t weights; + uint32_t output; + uint32_t add_output; + if (n < 3u) return 0; + matmul = node_bytes(model, n - 3u); + add = node_bytes(model, n - 2u); + softmax = node_bytes(model, n - 1u); + if (lwm_read_u16(matmul) != LW_OP_MATMUL || lwm_read_u16(add) != LW_OP_ADD || + lwm_read_u16(softmax) != LW_OP_SOFTMAX || lwm_read_u16(matmul + 2u) != 2u || + lwm_read_u16(add + 2u) != 2u || lwm_read_u16(softmax + 2u) != 1u) return 0; + activation = lwm_read_u32(matmul + 8u); + weights = lwm_read_u32(matmul + 12u); + output = lwm_read_u32(matmul + 40u); + add_output = lwm_read_u32(add + 40u); + if (lwm_read_u32(add + 8u) != output || session->tensors[weights].rank != 2u || + add_output != lwm_read_u32(softmax + 8u)) return 0; + tail->enabled = 1u; + tail->activation_value = activation; + tail->weight_tensor = weights; + tail->bias_tensor = lwm_read_u32(add + 12u); + tail->classes = (uint32_t)session->tensors[weights].dimensions[1]; + tail->inner = (uint32_t)session->tensors[weights].dimensions[0]; + tail->rows = (uint32_t)session->tensors[activation].dimensions[session->tensors[activation].rank - 2u]; + return 1; +} + +static uint64_t value_elements(const lw_x64_rec_value* value) { return value->bytes / sizeof(float); } + +static uint8_t choose_nchw_pointwise_kernel(uint32_t height, + uint32_t input_channels, + uint32_t output_channels, + uint32_t spatial) { + if (output_channels >= 8u && (output_channels & 7u) == 0u && spatial >= 8u && + ((height == 6u && input_channels == 512u && output_channels == 1024u) || + (height == 3u && ((input_channels == 768u && output_channels == 384u) || + (input_channels == 1536u && output_channels == 768u))))) { + return LW_X64_REC_NCHW_PW_FMA8; + } + return LW_X64_REC_NCHW_PW_FMA4; +} + +static uint8_t choose_pointwise_kernel(uint32_t pixels, uint32_t output_channels) { + if (pixels <= 1u && output_channels % 32u == 0u) return LW_X64_REC_PW_2X32; + if (pixels <= 1u && output_channels % 16u == 0u) return LW_X64_REC_PW_4X16; + return LW_X64_REC_PW_6X16; +} + +static int fused_gelu_temporaries_private(const lw_model* model, uint32_t node_index, + const lw_fused_gelu_match* match) { + uint32_t temporary; + if (model == NULL || match == NULL || node_index > model->info.node_count || + model->info.node_count - node_index < 5u) { + return 0; + } + for (temporary = 0u; temporary < 4u; ++temporary) { + uint32_t tensor = match->outputs[temporary]; + uint32_t node; + int32_t last_use = -1; + for (node = 0u; node < model->info.node_count; ++node) { + const uint8_t* current = node_bytes(model, node); + uint16_t input_count = lwm_read_u16(current + 2u); + uint16_t input_index; + for (input_index = 0u; input_index < input_count; ++input_index) { + if (lwm_read_u32(current + 8u + (size_t)input_index * sizeof(uint32_t)) == tensor) { + last_use = (int32_t)node; + } + } + } + for (node = 0u; node < model->info.output_count; ++node) { + if (lwm_read_u32(model->bytes + (size_t)model->output_offset + + (size_t)node * sizeof(uint32_t)) == tensor) { + last_use = (int32_t)model->info.node_count; + } + } + if (last_use != (int32_t)(node_index + temporary + 1u)) { + return 0; + } + } + return 1; +} + +/* Count how many node input slots reference `tensor`; returns UINT32_MAX when + * the tensor is also a model output (treat as externally consumed). */ +static uint32_t tensor_use_count(const lw_model* model, uint32_t tensor) { + uint32_t uses = 0u; + uint32_t node; + uint32_t output; + for (node = 0u; node < model->info.node_count; ++node) { + const uint8_t* current = node_bytes(model, node); + uint16_t input_count = lwm_read_u16(current + 2u); + uint16_t input_index; + for (input_index = 0u; input_index < input_count; ++input_index) { + if (lwm_read_u32(current + 8u + (size_t)input_index * sizeof(uint32_t)) == tensor) { + ++uses; + } + } + } + for (output = 0u; output < model->info.output_count; ++output) { + if (lwm_read_u32(model->bytes + (size_t)model->output_offset + + (size_t)output * sizeof(uint32_t)) == tensor) { + return UINT32_MAX; + } + } + return uses; +} + +/* True when `tensor` is consumed exactly once, by node `consumer`. */ +static int tensor_single_consumer(const lw_model* model, uint32_t tensor, + uint32_t consumer) { + const uint8_t* consumer_bytes; + uint16_t input_count; + uint16_t input_index; + uint32_t uses_here = 0u; + if (tensor_use_count(model, tensor) != 1u) { + return 0; + } + if (consumer >= model->info.node_count) { + return 0; + } + consumer_bytes = node_bytes(model, consumer); + input_count = lwm_read_u16(consumer_bytes + 2u); + for (input_index = 0u; input_index < input_count; ++input_index) { + if (lwm_read_u32(consumer_bytes + 8u + (size_t)input_index * sizeof(uint32_t)) == tensor) { + ++uses_here; + } + } + return uses_here == 1u; +} + +/* True when every consumer of `tensor` lies inside the node range + * [first, first + span) and the tensor is not a model output. The GELU chain + * reads its input twice (Div and the second Mul), so a plain single-consumer + * test would reject it. */ +static int tensor_consumed_within(const lw_model* model, uint32_t tensor, + uint32_t first, uint32_t span) { + uint32_t node; + uint32_t output; + int used = 0; + for (node = 0u; node < model->info.node_count; ++node) { + const uint8_t* current = node_bytes(model, node); + uint16_t input_count = lwm_read_u16(current + 2u); + uint16_t input_index; + for (input_index = 0u; input_index < input_count; ++input_index) { + if (lwm_read_u32(current + 8u + (size_t)input_index * sizeof(uint32_t)) == tensor) { + if (node < first || node >= first + span) { + return 0; + } + used = 1; + } + } + } + for (output = 0u; output < model->info.output_count; ++output) { + if (lwm_read_u32(model->bytes + (size_t)model->output_offset + + (size_t)output * sizeof(uint32_t)) == tensor) { + return 0; + } + } + return used; +} + +/* Conv epilogue folding: after a successfully lowered NHWC conv at semantic + * node `node_index`, greedily absorb a trailing channel-bias Add, a residual + * Add, and a five-node GELU chain into the conv's fused epilogue. Returns the + * number of semantic nodes consumed (1 = nothing folded). Folded operations + * are applied by the conv kernels in exactly the original + * accumulator -> +bias -> +residual -> activation order, so results stay + * bit-identical to the unfused op sequence. On a successful fold this also + * performs the lower_ops bookkeeping for the whole span: producer on the + * final output and last_use bumps for every consumed input. */ +static uint32_t fold_conv_epilogue(const lw_model* model, const lw_session* session, + lw_x64_rec_program* program, uint32_t limit, + uint32_t node_index, lw_x64_rec_op* op, + uint32_t physical) { + uint32_t current = lwm_read_u32(node_bytes(model, node_index) + 40u); + uint64_t current_bytes = program->values[current].bytes; + uint32_t span = 1u; + uint32_t next = node_index + 1u; + uint32_t residual_tensor = UINT32_MAX; + uint32_t final_out = current; + const float* post_bias = NULL; + uint16_t activation = LW_NHWC_ACT_NONE; + int allow_residual_gelu = op->kind != LW_X64_REC_OP_DEPTHWISE; + if (op->kind == LW_X64_REC_OP_DENSE) return 1u; + + /* 1. Channel-bias Add: conv_out + constant[channels]. */ + if (next < limit) { + const uint8_t* add = node_bytes(model, next); + if (lwm_read_u16(add) == LW_OP_ADD && lwm_read_u16(add + 2u) == 2u) { + uint32_t left = lwm_read_u32(add + 8u); + uint32_t right = lwm_read_u32(add + 12u); + uint32_t other = left == current ? right : right == current ? left : UINT32_MAX; + uint32_t add_out = lwm_read_u32(add + 40u); + if (other != UINT32_MAX && + program->values[other].constant_data != NULL && + program->values[other].bytes == + (uint64_t)op->data.conv.output_channels * sizeof(float) && + program->values[add_out].bytes == current_bytes && + tensor_single_consumer(model, current, next)) { + post_bias = program->values[other].constant_data; + current = add_out; + final_out = add_out; + ++span; + ++next; + } + } + } + /* 2. Residual Add: chain_out + non-constant tensor of the same size. */ + if (allow_residual_gelu && next < limit) { + const uint8_t* add = node_bytes(model, next); + if (lwm_read_u16(add) == LW_OP_ADD && lwm_read_u16(add + 2u) == 2u) { + uint32_t left = lwm_read_u32(add + 8u); + uint32_t right = lwm_read_u32(add + 12u); + uint32_t other = left == current ? right : right == current ? left : UINT32_MAX; + uint32_t add_out = lwm_read_u32(add + 40u); + if (other != UINT32_MAX && + program->values[other].constant_data == NULL && + program->values[other].bytes == current_bytes && + program->values[add_out].bytes == current_bytes && + tensor_single_consumer(model, current, next)) { + residual_tensor = other; + current = add_out; + final_out = add_out; + ++span; + ++next; + } + } + } + /* 3. Five-node GELU chain (Div -> Erf -> Add -> Mul -> Mul). */ + if (allow_residual_gelu && next + 5u <= limit) { + lw_fused_gelu_match match; + if (lw_match_fused_gelu(session, next, &match) && + match.inputs[0][0] == current && + fused_gelu_temporaries_private(model, next, &match) && + tensor_consumed_within(model, current, next, 5u)) { + activation = LW_NHWC_ACT_GELU; + final_out = match.outputs[4]; + span += 5u; + } + } + if (span == 1u) { + return 1u; + } + + op->data.conv.post_bias = post_bias; + if (residual_tensor != UINT32_MAX) { + op->data.conv.has_residual = 1u; + op->data.conv.residual_offset = program->values[residual_tensor].offset; + } + op->data.conv.activation = activation; + op->data.conv.output_offset = program->values[final_out].offset; + program->values[final_out].producer = (int32_t)physical; + { + uint32_t node; + for (node = node_index; node < node_index + span; ++node) { + const uint8_t* fused_bytes = node_bytes(model, node); + uint16_t fused_inputs = lwm_read_u16(fused_bytes + 2u); + uint16_t input_index; + for (input_index = 0u; input_index < fused_inputs; ++input_index) { + uint32_t input = lwm_read_u32( + fused_bytes + 8u + (size_t)input_index * sizeof(uint32_t)); + if (program->values[input].last_use < (int32_t)physical) { + program->values[input].last_use = (int32_t)physical; + } + } + } + } + return span; +} + +static lw_status compile_conv(const lw_model* model, const lw_session* session, + lw_x64_rec_program* program, const uint8_t* node, + lw_x64_rec_op* op, lw_error* error) { + const lw_runtime_tensor* input = &session->tensors[lwm_read_u32(node + 8u)]; + const lw_runtime_tensor* weight = &session->tensors[lwm_read_u32(node + 12u)]; + const lw_runtime_tensor* output = &session->tensors[lwm_read_u32(node + 40u)]; + const uint8_t* params = model->bytes + (size_t)lwm_read_u64(node + 56u); + uint32_t groups = lwm_read_u32(params + 4u); + uint32_t kh = (uint32_t)lwm_read_i32(params + 8u); + uint32_t kw = (uint32_t)lwm_read_i32(params + 12u); + uint32_t sh = (uint32_t)lwm_read_i32(params + 16u); + uint32_t sw = (uint32_t)lwm_read_i32(params + 20u); + uint64_t count; + float* packed; + op->data.conv.input_channels = (uint32_t)input->dimensions[1]; + op->data.conv.input_offset = program->values[lwm_read_u32(node + 8u)].offset; + op->data.conv.output_offset = program->values[lwm_read_u32(node + 40u)].offset; + op->data.conv.output_channels = (uint32_t)output->dimensions[1]; + op->data.conv.input_height = (uint32_t)input->dimensions[2]; + op->data.conv.input_width = (uint32_t)input->dimensions[3]; + op->data.conv.output_height = (uint32_t)output->dimensions[2]; + op->data.conv.output_width = (uint32_t)output->dimensions[3]; + op->data.conv.kernel_h = kh; op->data.conv.kernel_w = kw; + op->data.conv.weight_h = (uint32_t)weight->dimensions[2]; op->data.conv.weight_w = (uint32_t)weight->dimensions[3]; + op->data.conv.stride_h = sh; op->data.conv.stride_w = sw; + op->data.conv.pad_top = (uint32_t)lwm_read_i32(params + 32u); + op->data.conv.pad_left = (uint32_t)lwm_read_i32(params + 36u); + op->data.conv.pad_bottom = (uint32_t)lwm_read_i32(params + 40u); + op->data.conv.pad_right = (uint32_t)lwm_read_i32(params + 44u); + op->data.conv.groups = groups; + if (program->backend_layout == LW_X64_REC_BACKEND_NCHW) { + const uint32_t weight_index = lwm_read_u32(node + 12u); + const float* weights = constant_f32(model, weight_index); + if (weights == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "NCHW Conv weights are not constant"); + return LW_STATUS_INVALID_ARGUMENT; + } + op->data.conv.original_weights = weights; + if (lwm_read_u16(node + 2u) >= 3u) { + op->data.conv.bias = constant_f32(model, lwm_read_u32(node + 16u)); + } + if (groups == 1u && kh == 1u && kw == 1u && sh == 1u && sw == 1u) { + if (!lw_packed_conv1x1_weight_count(op->data.conv.input_channels, + op->data.conv.output_channels, &count)) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "NCHW pointwise shape is unsupported"); + return LW_STATUS_UNSUPPORTED; + } + packed = (float*)alloc_constant(program, count * sizeof(float)); + if (packed == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "NCHW pointwise pack allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + lw_pack_conv1x1_weights_f32(weights, op->data.conv.input_channels, + op->data.conv.output_channels, packed); + op->kind = LW_X64_REC_OP_POINTWISE_NCHW; + op->data.conv.kernel_kind = LW_X64_REC_CONV_POINTWISE; + op->data.conv.packed_weights = packed; + op->data.conv.nchw_pointwise_kernel = + choose_nchw_pointwise_kernel(op->data.conv.input_height, + op->data.conv.input_channels, + op->data.conv.output_channels, + op->data.conv.input_height * + op->data.conv.input_width); + return LW_STATUS_OK; + } + if (groups == 1u && kh == 3u && kw == 3u && sh == 2u && sw == 2u && + op->data.conv.pad_top == 1u && op->data.conv.pad_left == 1u && + op->data.conv.pad_bottom == 1u && op->data.conv.pad_right == 1u && + lw_packed_conv3x3_stride2_weight_count(op->data.conv.input_channels, + op->data.conv.output_channels, &count)) { + packed = (float*)alloc_constant(program, count * sizeof(float)); + if (packed == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "NCHW stem pack allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + lw_pack_conv3x3_stride2_weights_f32(weights, op->data.conv.input_channels, + op->data.conv.output_channels, packed); + op->kind = LW_X64_REC_OP_STEM_NCHW; + op->data.conv.kernel_kind = LW_X64_REC_CONV_DENSE; + op->data.conv.packed_weights = packed; + return LW_STATUS_OK; + } + if (groups == op->data.conv.input_channels && + groups == op->data.conv.output_channels && + weight->dimensions[1] == 1u && + ((kh == 3u && kw == 3u && + op->data.conv.pad_top == 1u && op->data.conv.pad_left == 1u && + op->data.conv.pad_bottom == 1u && op->data.conv.pad_right == 1u && + ((sh == 1u && sw == 1u) || (sh == 2u && sw == 1u))) || + (kh == 1u && kw == 5u && sh == 1u && sw == 1u && + op->data.conv.pad_top == 0u && op->data.conv.pad_left == 2u && + op->data.conv.pad_bottom == 0u && op->data.conv.pad_right == 2u))) { + op->kind = LW_X64_REC_OP_DEPTHWISE_NCHW; + op->data.conv.kernel_kind = LW_X64_REC_CONV_DEPTHWISE; + return LW_STATUS_OK; + } + char message[192]; (void)snprintf(message, sizeof(message), "Tiny NCHW Conv shape is unsupported: groups=%u in=%u out=%u kernel=%ux%u stride=%ux%u pad=%u,%u,%u,%u", groups, op->data.conv.input_channels, op->data.conv.output_channels, kh, kw, sh, sw, op->data.conv.pad_top, op->data.conv.pad_left, op->data.conv.pad_bottom, op->data.conv.pad_right); lw_set_error(error, LW_STATUS_UNSUPPORTED, message); + return LW_STATUS_UNSUPPORTED; + } + if (groups == 1u && kh == 1u && kw == 1u && sh == 1u && sw == 1u && + lw_nhwc_dense_packed_weight_count(op->data.conv.input_channels, + op->data.conv.output_channels, 1u, 1u, &count)) { + op->kind = LW_X64_REC_OP_POINTWISE; + op->data.conv.kernel_kind = LW_X64_REC_CONV_POINTWISE; + op->data.conv.pointwise_kernel = choose_pointwise_kernel( + op->data.conv.input_height * op->data.conv.input_width, + op->data.conv.output_channels); + } else if (groups == op->data.conv.input_channels && groups == op->data.conv.output_channels && + weight->dimensions[1] == 1 && + lw_nhwc_depthwise_packed_weight_count(op->data.conv.input_channels, kh, kw, &count)) { + op->kind = LW_X64_REC_OP_DEPTHWISE; + op->data.conv.kernel_kind = LW_X64_REC_CONV_DEPTHWISE; + } else if (groups == 1u && + lw_nhwc_dense_packed_weight_count(op->data.conv.input_channels, + op->data.conv.output_channels, kh, kw, &count)) { + op->kind = LW_X64_REC_OP_DENSE; + op->data.conv.kernel_kind = LW_X64_REC_CONV_DENSE; + } else { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "Tiny REC Conv shape is not NHWC-lowerable"); + return LW_STATUS_UNSUPPORTED; + } + op->data.conv.original_weights = constant_f32(model, lwm_read_u32(node + 12u)); + packed = (float*)borrow_conv_pack(program, op, count * sizeof(float)); + if (packed == NULL) { + packed = (float*)alloc_constant(program, count * sizeof(float)); + if (packed == NULL) { lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "Conv packed weight allocation failed"); return LW_STATUS_OUT_OF_MEMORY; } + if (op->data.conv.kernel_kind == LW_X64_REC_CONV_DEPTHWISE) + lw_pack_nhwc_depthwise_f32(op->data.conv.original_weights, op->data.conv.input_channels, kh, kw, packed); + else + lw_pack_nhwc_dense_f32(op->data.conv.original_weights, op->data.conv.input_channels, + op->data.conv.output_channels, kh, kw, packed); + } + op->data.conv.packed_weights = packed; + if (lwm_read_u16(node + 2u) >= 3u) op->data.conv.bias = constant_f32(model, lwm_read_u32(node + 16u)); + op->data.conv.scratch_bytes = 0u; + if (op->kind == LW_X64_REC_OP_DENSE) { + op->data.conv.dense_kc = LW_NHWC_DENSE_KC; + lw_nhwc_dense_desc desc; + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; desc.input_channels = op->data.conv.input_channels; + desc.input_height = op->data.conv.input_height; desc.input_width = op->data.conv.input_width; + desc.output_channels = op->data.conv.output_channels; desc.output_height = op->data.conv.output_height; + desc.output_width = op->data.conv.output_width; desc.kernel_h = kh; desc.kernel_w = kw; + desc.stride_h = sh; desc.stride_w = sw; desc.pad_top = op->data.conv.pad_top; desc.pad_left = op->data.conv.pad_left; + desc.pad_bottom = op->data.conv.pad_bottom; desc.pad_right = op->data.conv.pad_right; + desc.dense_kc = op->data.conv.dense_kc; + if (!lw_nhwc_dense_scratch_bytes(&desc, &op->data.conv.scratch_bytes)) { + op->data.conv.scalar_fallback = 1u; + op->data.conv.scratch_bytes = 0u; + } + if (op->data.conv.scratch_bytes > program->scratch_bytes) program->scratch_bytes = op->data.conv.scratch_bytes; + } + return LW_STATUS_OK; +} + +static lw_status compile_node(const lw_model* model, const lw_session* session, + lw_x64_rec_program* program, uint32_t node_index, + lw_x64_rec_op* op, lw_error* error) { + const uint8_t* node = node_bytes(model, node_index); + const uint16_t semantic = lwm_read_u16(node); + uint32_t input_index = lwm_read_u32(node + 8u); + uint32_t output_index = lwm_read_u32(node + 40u); + const lw_runtime_tensor* input = &session->tensors[input_index]; + const lw_runtime_tensor* output = &session->tensors[output_index]; + const lw_x64_rec_value* in_value = &program->values[input_index]; + lw_x64_rec_value* out_value = &program->values[output_index]; + const uint8_t* params = lwm_read_u64(node + 56u) == 0u ? NULL : model->bytes + (size_t)lwm_read_u64(node + 56u); + uint64_t elements = value_elements(out_value); + op->semantic_begin = node_index; op->semantic_count = 1u; + switch (semantic) { + case LW_OP_CONV: return compile_conv(model, session, program, node, op, error); + case LW_OP_RELU: op->kind = LW_X64_REC_OP_RELU; break; + case LW_OP_ERF: op->kind = LW_X64_REC_OP_ERF; break; + case LW_OP_HARD_SIGMOID: op->kind = LW_X64_REC_OP_HARD_SIGMOID; op->data.unary.alpha = read_f32(params + 4u); op->data.unary.beta = read_f32(params + 8u); break; + case LW_OP_SIGMOID: op->kind = LW_X64_REC_OP_SIGMOID; break; + case LW_OP_SQRT: op->kind = LW_X64_REC_OP_SQRT; break; + case LW_OP_BATCH_NORMALIZATION: { + uint32_t c = (uint32_t)input->dimensions[1]; + const float* scale = constant_f32(model, lwm_read_u32(node + 12u)); + const float* bias = constant_f32(model, lwm_read_u32(node + 16u)); + const float* mean = constant_f32(model, lwm_read_u32(node + 20u)); + const float* variance = constant_f32(model, lwm_read_u32(node + 24u)); + float epsilon = read_f32(params + 4u); + const lw_x64_rec_op* source = find_packed_source_op(program, node_index); + float* mul; + float* add; + uint32_t j; + if (source != NULL && source->kind == LW_X64_REC_OP_AFFINE && + source->data.affine.channels == c && source->data.affine.epsilon == epsilon && + source->data.affine.scale == scale && source->data.affine.bias == bias && + source->data.affine.mean == mean && source->data.affine.variance == variance && + source->data.affine.mul != NULL && source->data.affine.add != NULL) { + mul = (float*)source->data.affine.mul; + add = (float*)source->data.affine.add; + note_borrow(program, (uint64_t)c * sizeof(float)); + note_borrow(program, (uint64_t)c * sizeof(float)); + } else { + mul = (float*)alloc_constant(program, (uint64_t)c * sizeof(float)); + add = (float*)alloc_constant(program, (uint64_t)c * sizeof(float)); + if (mul == NULL || add == NULL) return LW_STATUS_OUT_OF_MEMORY; + for (j = 0u; j < c; ++j) { + float factor = scale[j] / sqrtf(variance[j] + epsilon); + mul[j] = factor; + add[j] = bias[j] - mean[j] * factor; + } + } + op->kind = LW_X64_REC_OP_AFFINE; + op->data.affine.input_offset = in_value->offset; + op->data.affine.output_offset = out_value->offset; + op->data.affine.mul = mul; + op->data.affine.add = add; + op->data.affine.scale = scale; + op->data.affine.bias = bias; + op->data.affine.mean = mean; + op->data.affine.variance = variance; + op->data.affine.epsilon = epsilon; + op->data.affine.channel_major = (uint8_t)(program->backend_layout == LW_X64_REC_BACKEND_NCHW || input->rank != 4u); + op->data.affine.pixels = (uint32_t)(value_elements(in_value) / c); + op->data.affine.channels = c; + break; + } + case LW_OP_ADD: case LW_OP_MUL: case LW_OP_DIV: case LW_OP_SUB: case LW_OP_POW: + op->kind = semantic == LW_OP_ADD ? LW_X64_REC_OP_ADD : semantic == LW_OP_MUL ? LW_X64_REC_OP_MUL : semantic == LW_OP_DIV ? LW_X64_REC_OP_DIV : semantic == LW_OP_SUB ? LW_X64_REC_OP_SUB : semantic == LW_OP_POW ? LW_X64_REC_OP_POW : LW_X64_REC_OP_GENERIC_UNSUPPORTED; op->data.binary.operation=semantic; op->data.binary.element_count=elements; op->data.binary.output_offset=out_value->offset; op->data.binary.left_offset=program->values[input_index].offset; op->data.binary.left_constant=program->values[input_index].constant_data; op->data.binary.right_offset=program->values[lwm_read_u32(node+12u)].offset; op->data.binary.right_constant=program->values[lwm_read_u32(node+12u)].constant_data; op->data.binary.broadcast_kind=(program->values[lwm_read_u32(node+12u)].bytes==sizeof(float))?LW_X64_REC_BROADCAST_RIGHT_SCALAR:program->values[lwm_read_u32(node+12u)].bytes==out_value->bytes?LW_X64_REC_BROADCAST_SAME:((program->values[lwm_read_u32(node+12u)].bytes == (uint64_t)rec_binary_channel_extent(model, session, out_value, lwm_read_u32(node + 40u)) * sizeof(float))?LW_X64_REC_BROADCAST_RIGHT_CHANNEL:((program->values[input_index].bytes == (uint64_t)rec_binary_channel_extent(model, session, out_value, lwm_read_u32(node + 40u)) * sizeof(float))?LW_X64_REC_BROADCAST_LEFT_CHANNEL:LW_X64_REC_BROADCAST_GENERAL)); memcpy(op->data.binary.dimensions, out_value->dimensions, sizeof(op->data.binary.dimensions)); op->data.binary.channels = rec_binary_channel_extent(model, session, out_value, lwm_read_u32(node + 40u)); op->data.binary.pixels = op->data.binary.channels != 0u ? (uint32_t)(elements / op->data.binary.channels) : 0u; + if (program->backend_layout == LW_X64_REC_BACKEND_NCHW && out_value->rank == 4u) { + op->data.binary.channels = (uint32_t)out_value->dimensions[1]; + op->data.binary.pixels = op->data.binary.channels != 0u ? + (uint32_t)(elements / op->data.binary.channels) : 0u; + if (program->values[lwm_read_u32(node + 12u)].bytes == + (uint64_t)op->data.binary.channels * sizeof(float)) { + op->data.binary.broadcast_kind = LW_X64_REC_BROADCAST_RIGHT_CHANNEL; + op->data.binary.channel_major_nchw = 1u; + } else if (program->values[input_index].bytes == + (uint64_t)op->data.binary.channels * sizeof(float)) { + op->data.binary.broadcast_kind = LW_X64_REC_BROADCAST_LEFT_CHANNEL; + op->data.binary.channel_major_nchw = 1u; + } + } + /* Shape-exact pixel-scalar broadcast ([1,W,C] op [1,W,1]). Must be + * evaluated before trusting the byte-based channel detection: when + * W == channels the byte counts collide. */ + if (out_value->rank >= 2u) { + /* One scalar per pixel broadcast over the last axis (e.g. + * [1,W,C] op [1,W,1]). Byte counts alone are ambiguous when + * pixels == channels, so compare the dimension tuples: same + * rank, equal leading dims, trailing right dim of 1. The + * general executor path only handles constant right + * operands, so route computed ones here. */ + const lw_runtime_tensor* right_tensor = + &session->tensors[lwm_read_u32(node + 12u)]; + uint32_t out_rank = out_value->rank; + int pixel_scalar = right_tensor->rank == out_rank && + right_tensor->dimensions[out_rank - 1u] == 1 && + out_value->dimensions[out_rank - 1u] > 1; + uint32_t axis; + for (axis = 0u; axis + 1u < out_rank && pixel_scalar; ++axis) { + if (right_tensor->dimensions[axis] != out_value->dimensions[axis]) { + pixel_scalar = 0; + } + } + if (pixel_scalar) { + op->data.binary.broadcast_kind = LW_X64_REC_BROADCAST_RIGHT_PIXEL_SCALAR; + } + } + if (program->backend_layout != LW_X64_REC_BACKEND_NCHW && out_value->rank == 4u && + op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_SAME) { + /* SAME assumes both operands share one physical layout. The + * attention chain holds canonical-order rank-4 tensors while + * the conv chain is NHWC, so a cross-chain elementwise op + * (e.g. positional-gate add) must remap the NHWC side. */ + const lw_runtime_tensor* out_tensor4 = &session->tensors[lwm_read_u32(node + 40u)]; + int left_nhwc = rec_value_is_nhwc_physical(model, session, input_index); + int right_nhwc = rec_value_is_nhwc_physical(model, session, lwm_read_u32(node + 12u)); + if (left_nhwc != right_nhwc) { + op->data.binary.mixed_nhwc = (uint8_t)(right_nhwc != 0 ? 1u : 2u); + memcpy(op->data.binary.dimensions, out_tensor4->dimensions, + sizeof(op->data.binary.dimensions)); + } + } + break; + case LW_OP_REDUCE_MEAN: { + uint32_t axes_count = params != NULL ? (uint32_t)lwm_read_u16(params + 2u) : 0u; + int32_t axis0 = axes_count > 0u ? lwm_read_i32(params + 12u) : 0; + int32_t axis1 = axes_count > 1u ? lwm_read_i32(params + 16u) : 0; + op->kind=LW_X64_REC_OP_REDUCE_MEAN; op->data.reduce.input_offset=in_value->offset; op->data.reduce.output_offset=out_value->offset; op->data.reduce.batch=(uint32_t)input->dimensions[0]; op->data.reduce.height=(uint32_t)input->dimensions[2]; op->data.reduce.width=(uint32_t)input->dimensions[3]; op->data.reduce.channels=(uint32_t)input->dimensions[1]; + if (input->rank == 4u && axes_count == 2u && axis0 == 2 && axis1 == 3) { + op->data.reduce.general = 0u; + } else { + uint32_t axis_index; + op->data.reduce.general = 1u; + op->data.reduce.input_rank = input->rank; + op->data.reduce.output_rank = output->rank; + op->data.reduce.axes_count = axes_count; + op->data.reduce.keep_dimensions = params != NULL ? lwm_read_u32(params + 4u) : 1u; + op->data.reduce.no_op_with_empty_axes = params != NULL ? lwm_read_u32(params + 8u) : 0u; + for (axis_index = 0u; axis_index < LW_MAX_DIMS; ++axis_index) { + op->data.reduce.axes[axis_index] = axis_index < axes_count ? + lwm_read_i32(params + 12u + axis_index * 4u) : 0; + } + memcpy(op->data.reduce.input_dimensions, input->dimensions, sizeof(op->data.reduce.input_dimensions)); + memcpy(op->data.reduce.output_dimensions, output->dimensions, sizeof(op->data.reduce.output_dimensions)); + } + break; + } + case LW_OP_AVERAGE_POOL: case LW_OP_MAX_POOL: op->kind=semantic==LW_OP_MAX_POOL?LW_X64_REC_OP_MAX_POOL:LW_X64_REC_OP_AVG_POOL; op->data.pool.input_offset=in_value->offset; op->data.pool.output_offset=out_value->offset; op->data.pool.input_dimensions[0]=input->dimensions[0]; + op->data.pool.input_dimensions[1]=program->backend_layout == LW_X64_REC_BACKEND_NCHW ? input->dimensions[1] : input->dimensions[2]; + op->data.pool.input_dimensions[2]=program->backend_layout == LW_X64_REC_BACKEND_NCHW ? input->dimensions[2] : input->dimensions[3]; + op->data.pool.input_dimensions[3]=program->backend_layout == LW_X64_REC_BACKEND_NCHW ? input->dimensions[3] : input->dimensions[1]; + op->data.pool.output_dimensions[0]=output->dimensions[0]; + op->data.pool.output_dimensions[1]=program->backend_layout == LW_X64_REC_BACKEND_NCHW ? output->dimensions[1] : output->dimensions[2]; + op->data.pool.output_dimensions[2]=program->backend_layout == LW_X64_REC_BACKEND_NCHW ? output->dimensions[2] : output->dimensions[3]; + op->data.pool.output_dimensions[3]=program->backend_layout == LW_X64_REC_BACKEND_NCHW ? output->dimensions[3] : output->dimensions[1]; op->data.pool.kernel[0]=lwm_read_i32(params+8u); op->data.pool.kernel[1]=lwm_read_i32(params+12u); op->data.pool.strides[0]=lwm_read_i32(params+16u); op->data.pool.strides[1]=lwm_read_i32(params+20u); op->data.pool.pads[0]=lwm_read_i32(params+24u); op->data.pool.pads[1]=lwm_read_i32(params+28u); op->data.pool.pads[2]=lwm_read_i32(params+32u); op->data.pool.pads[3]=lwm_read_i32(params+36u); op->data.pool.count_include_pad=(uint8_t)lwm_read_u32(params+44u); op->data.pool.is_max=(uint8_t)(semantic==LW_OP_MAX_POOL); break; + case LW_OP_MATMUL: { + const lw_runtime_tensor* w = &session->tensors[lwm_read_u32(node + 12u)]; + const lw_runtime_tensor* out_tensor = &session->tensors[lwm_read_u32(node + 40u)]; + uint64_t count = 0u; + uint32_t batch = 1u; + uint32_t bi; + int packable; + for (bi = 0u; bi + 2u < input->rank; ++bi) { + batch *= (uint32_t)input->dimensions[bi]; + } + op->kind = LW_X64_REC_OP_MATMUL; + op->data.matmul.input_offset = in_value->offset; + op->data.matmul.output_offset = out_value->offset; + op->data.matmul.weights_offset = program->values[lwm_read_u32(node + 12u)].offset; + op->data.matmul.weights = program->values[lwm_read_u32(node + 12u)].constant_data; + op->data.matmul.batch = batch; + op->data.matmul.rows = (uint32_t)input->dimensions[input->rank - 2u]; + op->data.matmul.inner = (uint32_t)input->dimensions[input->rank - 1u]; + op->data.matmul.columns = w->rank >= 2u ? (uint32_t)w->dimensions[w->rank - 1u] : 0u; + op->data.matmul.general = 0u; + op->data.matmul.input_rank = input->rank; + op->data.matmul.weights_rank = w->rank; + op->data.matmul.output_rank = out_tensor->rank; + memcpy(op->data.matmul.input_dimensions, input->dimensions, sizeof(op->data.matmul.input_dimensions)); + memcpy(op->data.matmul.weights_dimensions, w->dimensions, sizeof(op->data.matmul.weights_dimensions)); + memcpy(op->data.matmul.output_dimensions, out_tensor->dimensions, sizeof(op->data.matmul.output_dimensions)); + /* Mirror the canonical session's packed-matmul eligibility exactly + * so both paths accumulate in the same order (bit-identical). */ + packable = op->data.matmul.weights != NULL && w->rank == 2u && + out_tensor->rank == input->rank && + op->data.matmul.rows >= 4u && op->data.matmul.rows % 4u == 0u && + op->data.matmul.inner >= 64u && op->data.matmul.columns >= 1024u && + w->dimensions[0] == (int32_t)op->data.matmul.inner && + out_tensor->dimensions[out_tensor->rank - 2u] == (int32_t)op->data.matmul.rows && + out_tensor->dimensions[out_tensor->rank - 1u] == (int32_t)op->data.matmul.columns && + lw_packed_matmul_weight_count(op->data.matmul.inner, op->data.matmul.columns, &count); + if (packable) { + op->data.matmul.packed_weights = (float*)borrow_matmul_pack(program, op, count * sizeof(float)); + if (op->data.matmul.packed_weights == NULL) { + op->data.matmul.packed_weights = (float*)alloc_constant(program, count * sizeof(float)); + if (op->data.matmul.packed_weights == NULL) return LW_STATUS_OUT_OF_MEMORY; + lw_pack_matmul_weights_f32(op->data.matmul.weights, op->data.matmul.inner, + op->data.matmul.columns, op->data.matmul.packed_weights); + } + } else { + op->data.matmul.general = 1u; + } + break; } + case LW_OP_TRANSPOSE: op->kind=LW_X64_REC_OP_TRANSPOSE; op->data.transpose.input_offset=in_value->offset; op->data.transpose.output_offset=out_value->offset; op->data.transpose.rank=input->rank; memcpy(op->data.transpose.input_dimensions,input->dimensions,sizeof(op->data.transpose.input_dimensions)); memcpy(op->data.transpose.output_dimensions,output->dimensions,sizeof(op->data.transpose.output_dimensions)); for(uint32_t j=0;jrank && jdata.transpose.permutation[j]=lwm_read_i32(params+4u+j*4u); break; + case LW_OP_SOFTMAX: op->kind=LW_X64_REC_OP_SOFTMAX; op->data.softmax.input_offset=in_value->offset; op->data.softmax.output_offset=out_value->offset; op->data.softmax.axis=lwm_read_i32(params+4u); op->data.softmax.rank=input->rank; memcpy(op->data.softmax.dimensions,input->dimensions,sizeof(op->data.softmax.dimensions)); break; + case LW_OP_CONCAT: { + uint16_t concat_input_count = lwm_read_u16(node + 2u); + int32_t concat_axis = params != NULL ? lwm_read_i32(params + 4u) : 0; + uint16_t concat_slot; + /* Only the channel axis (NCHW axis 1) of rank-4 feature maps is + * supported: stored NHWC, it is a per-pixel channel copy. Other + * shapes keep the model on the canonical executor. */ + if (params == NULL || concat_input_count < 2u || + concat_input_count > LWM_V0_MAX_NODE_INPUTS || output->rank != 4u || + concat_axis != 1 || program->backend_layout != LW_X64_REC_BACKEND_NHWC) { + op->kind = LW_X64_REC_OP_GENERIC_UNSUPPORTED; + break; + } + op->kind = LW_X64_REC_OP_CONCAT; + op->data.concat.input_count = concat_input_count; + op->data.concat.output_offset = out_value->offset; + op->data.concat.axis = concat_axis; + op->data.concat.output_channels = (uint32_t)output->dimensions[1]; + op->data.concat.pixels = (uint32_t)((uint64_t)output->dimensions[2] * + (uint64_t)output->dimensions[3]); + for (concat_slot = 0u; concat_slot < concat_input_count; ++concat_slot) { + uint32_t concat_input = lwm_read_u32(node + 8u + (size_t)concat_slot * 4u); + op->data.concat.input_offsets[concat_slot] = program->values[concat_input].offset; + op->data.concat.input_channels[concat_slot] = + (uint32_t)session->tensors[concat_input].dimensions[1]; + } + break; + } + case LW_OP_SLICE: { + uint32_t slice_count = params != NULL ? (uint32_t)lwm_read_u16(params + 2u) : 0u; + uint32_t slice_index; + /* Rank-4 values are physically NHWC here, so only slice + * canonical-layout tensors (rank != 4) via the scalar kernel. */ + if (params == NULL || slice_count == 0u || slice_count > LW_MAX_DIMS || + input->rank > LW_MAX_DIMS || input->rank == 4u) { + op->kind = LW_X64_REC_OP_GENERIC_UNSUPPORTED; + break; + } + op->kind = LW_X64_REC_OP_SLICE; + op->data.slice.input_offset = in_value->offset; + op->data.slice.output_offset = out_value->offset; + op->data.slice.rank = input->rank; + op->data.slice.output_rank = output->rank; + op->data.slice.slice_count = slice_count; + memcpy(op->data.slice.input_dimensions, input->dimensions, sizeof(op->data.slice.input_dimensions)); + memcpy(op->data.slice.output_dimensions, output->dimensions, sizeof(op->data.slice.output_dimensions)); + for (slice_index = 0u; slice_index < LW_MAX_DIMS; ++slice_index) { + op->data.slice.starts[slice_index] = slice_index < slice_count ? lwm_read_i32(params + 4u + slice_index * 4u) : 0; + op->data.slice.ends[slice_index] = slice_index < slice_count ? lwm_read_i32(params + 36u + slice_index * 4u) : 0; + op->data.slice.axes[slice_index] = slice_index < slice_count ? lwm_read_i32(params + 68u + slice_index * 4u) : 0; + op->data.slice.steps[slice_index] = slice_index < slice_count ? lwm_read_i32(params + 100u + slice_index * 4u) : 0; + } + break; + } + case LW_OP_RESIZE: op->kind=LW_X64_REC_OP_GENERIC_UNSUPPORTED; break; + default: op->kind=LW_X64_REC_OP_GENERIC_UNSUPPORTED; break; + } + if (op->kind == LW_X64_REC_OP_RELU || op->kind == LW_X64_REC_OP_ERF || op->kind == LW_X64_REC_OP_HARD_SIGMOID || op->kind == LW_X64_REC_OP_SIGMOID || op->kind == LW_X64_REC_OP_SQRT) { op->data.unary.input_offset=in_value->offset; op->data.unary.output_offset=out_value->offset; op->data.unary.element_count=elements; op->data.unary.rank=out_value->rank; memcpy(op->data.unary.dimensions,out_value->dimensions,sizeof(op->data.unary.dimensions)); } + return LW_STATUS_OK; +} + +static uint32_t resolve_alias_root(const lw_x64_rec_program* program, uint32_t index) { + uint32_t guard = 0u; + while (program->values[index].alias != 0u && guard++ < program->value_count) { + index = program->values[index].alias_of; + } + return index; +} + +typedef struct arena_candidate { + uint32_t value_index; + uint64_t bytes; + int32_t first; + int32_t last; + uint64_t offset; +} arena_candidate; + +static int candidate_before(const arena_candidate* a, const arena_candidate* b) { + if (a->bytes != b->bytes) return a->bytes > b->bytes; + if (a->first != b->first) return a->first < b->first; + return a->value_index < b->value_index; +} + +/* Offline greedy-by-size interval placement: each value gets a lifetime + * [first,last] in physical-op space; values whose intervals overlap cannot + * share an offset. Values never produced by a physical op (GELU fusion + * temporaries, the CTC-excluded tail tensors, dead model tensors) get no slot, + * which also removes their storage. */ +static lw_status plan_arena_offsets(lw_x64_rec_program* program, lw_error* error) { + arena_candidate* candidates; + uint32_t count = 0u; + uint32_t i; + uint64_t arena_bytes = 0u; + candidates = (arena_candidate*)calloc(program->value_count, sizeof(*candidates)); + if (candidates == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "REC arena plan allocation failed"); + return LW_STATUS_OUT_OF_MEMORY; + } + if (program->ctc_fused) { + /* The CTC tail reads the activation after the op loop; keep it live + * until the end so no other value may overlap it. */ + lw_x64_rec_value* activation = &program->values[program->ctc.activation_value]; + if (activation->last_use < (int32_t)program->op_count) { + activation->last_use = (int32_t)program->op_count; + } + } + for (i = 0u; i < program->value_count; ++i) { + const lw_x64_rec_value* value = &program->values[i]; + int32_t first; + int32_t last; + uint32_t a; + if (value->alias != 0u || value->constant_data != NULL) continue; + if ((int32_t)i == (int32_t)program->input_value) { + /* The caller rewrites the input between runs, so it must never + * share its offset with any value produced inside the program. */ + first = 0; + last = (int32_t)program->op_count; + } else if (value->producer >= 0) { + first = value->producer; + last = value->last_use > value->producer ? value->last_use : value->producer; + } else { + continue; /* never produced: fusion temporaries, CTC tail, dead tensors */ + } + for (a = 0u; a < program->value_count; ++a) { + const lw_x64_rec_value* alias = &program->values[a]; + if (alias->alias != 0u && resolve_alias_root(program, a) == i && + alias->last_use > last) { + last = alias->last_use; + } + } + candidates[count].value_index = i; + candidates[count].bytes = value->bytes; + candidates[count].first = first; + candidates[count].last = last; + ++count; + } + for (i = 1u; i < count; ++i) { + arena_candidate current = candidates[i]; + uint32_t j = i; + while (j > 0u && candidate_before(¤t, &candidates[j - 1u])) { + candidates[j] = candidates[j - 1u]; + --j; + } + candidates[j] = current; + } + for (i = 0u; i < count; ++i) { + arena_candidate* current = &candidates[i]; + uint64_t offset = 0u; + int collided = 1; + while (collided) { + uint32_t p; + collided = 0; + for (p = 0u; p < i; ++p) { + const arena_candidate* placed = &candidates[p]; + if (placed->first > current->last || current->first > placed->last) continue; + if (offset < placed->offset + placed->bytes && + placed->offset < offset + current->bytes) { + offset = lw_x64_rec_arena_align(placed->offset + placed->bytes, 64u); + if (offset == UINT64_MAX) { + free(candidates); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, + "REC arena plan overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + collided = 1; + break; + } + } + } + offset = lw_x64_rec_arena_align(offset, 64u); + if (offset == UINT64_MAX || current->bytes > UINT64_MAX - offset) { + free(candidates); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "REC arena plan overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + current->offset = offset; + if (offset + current->bytes > arena_bytes) arena_bytes = offset + current->bytes; + program->values[current->value_index].offset = offset; + } + for (i = 0u; i < program->value_count; ++i) { + if (program->values[i].alias != 0u) { + program->values[i].offset = + program->values[resolve_alias_root(program, i)].offset; + } + } + program->arena_bytes = arena_bytes; + free(candidates); + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +/* Reset program state owned by a lowering pass so the second pass restarts + * from the same zero counters and empty constant list. */ +static void reset_lowering(lw_x64_rec_program* program, uint32_t limit) { + uint32_t i; + for (i = 0u; i < program->packed_constant_count; ++i) free(program->constants[i].data); + free(program->constants); + program->constants = NULL; + program->packed_constant_count = 0u; + program->owned_constant_bytes = 0u; + program->borrowed_constant_bytes = 0u; + program->borrowed_constant_count = 0u; + memset(program->ops, 0, (size_t)limit * sizeof(*program->ops)); + program->op_count = 0u; + program->semantic_consumed = 0u; + program->semantic_elided = 0u; + program->scratch_bytes = 0u; + program->unsupported_nodes = 0u; +} + +static lw_x64_rec_compile_result lower_ops(const lw_model* model, const lw_session* session, + lw_x64_rec_program* program, uint32_t limit, + lw_error* error); + +#if defined(LW_EXPERIMENTAL_REC_FFN_TILING) +/* Discover pairs before placing tensors, then reproduce the same flags after + * offsets are embedded. Scheduling the second op early extends the first + * input's lifetime: the projected output must not reuse unread input pixels. */ +static void mark_ffn_pairs(lw_x64_rec_program* program) { + uint32_t i; + if (program->backend_layout != LW_X64_REC_BACKEND_NHWC || !program->ctc_fused) return; + for (i = 0u; i + 1u < program->op_count; ++i) { + lw_x64_rec_op* first = &program->ops[i]; + const lw_x64_rec_op* second = &program->ops[i + 1u]; + const lw_x64_rec_conv_op* a = &first->data.conv; + const lw_x64_rec_conv_op* b = &second->data.conv; + uint32_t middle, input, output; + int is_output = 0; + if (first->kind != LW_X64_REC_OP_POINTWISE || + second->kind != LW_X64_REC_OP_POINTWISE || + a->scalar_fallback || b->scalar_fallback || a->has_residual || + a->activation != LW_NHWC_ACT_GELU || b->activation != LW_NHWC_ACT_NONE || + a->output_channels != b->input_channels || + a->input_height != b->input_height || a->input_width != b->input_width || + a->input_height != a->output_height || a->input_width != a->output_width || + b->input_height != b->output_height || b->input_width != b->output_width || + first->semantic_count == 0u) continue; + middle = lwm_read_u32(node_bytes(program->model, + first->semantic_begin + first->semantic_count - 1u) + 40u); + if (middle != lwm_read_u32(node_bytes(program->model, second->semantic_begin) + 8u) || + !tensor_single_consumer(program->model, middle, second->semantic_begin) || + program->values[middle].alias || + program->values[middle].last_use != (int32_t)(i + 1u)) continue; + for (output = 0u; output < program->model->info.output_count; ++output) { + if (middle == lwm_read_u32(program->model->bytes + + (size_t)program->model->output_offset + (size_t)output * sizeof(uint32_t))) { + is_output = 1; + } + } + if (is_output || middle == program->ctc.activation_value) continue; + input = lwm_read_u32(node_bytes(program->model, first->semantic_begin) + 8u); + if (program->values[input].last_use < (int32_t)(i + 1u)) + program->values[input].last_use = (int32_t)(i + 1u); + first->flags |= LW_X64_REC_OP_FFN_BEGIN; + ++i; + } +} +#endif + +static lw_x64_rec_compile_result rec_backend_compile_input_impl(const lw_model* model, uint32_t input_height, + uint32_t target_width, + lw_x64_rec_compile_strategy strategy, + uint32_t allow_ctc_tail, + const lw_x64_rec_program* packed_source, + lw_x64_rec_program** out_program, lw_error* error) { + lw_session* session=NULL; lw_x64_rec_program* program=NULL; lw_model_info info; lw_status status; uint32_t i, limit; lw_x64_rec_compile_result lowered; + if(out_program==NULL||model==NULL||target_width==0u||target_width>INT32_MAX||input_height==0u||input_height>INT32_MAX){lw_set_error(error,LW_STATUS_INVALID_ARGUMENT,"REC backend arguments are invalid");return LW_X64_REC_COMPILE_INVALID_GRAPH;} *out_program=NULL; + if (packed_source != NULL && (packed_source->model != model || packed_source->backend_layout != (uint8_t)strategy || packed_source->packed_source != NULL || packed_source->ops == NULL)) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "REC packed source is incompatible"); + return LW_X64_REC_COMPILE_INVALID_GRAPH; + } + lw_model_info_init(&info); status=lw_model_get_info(model,&info); if(status!=LW_STATUS_OK)return LW_X64_REC_COMPILE_INVALID_GRAPH; status=make_shape_session(model,input_height,target_width,&session,error); if(status!=LW_STATUS_OK)return status==LW_STATUS_OUT_OF_MEMORY?LW_X64_REC_COMPILE_OUT_OF_MEMORY:LW_X64_REC_COMPILE_INVALID_GRAPH; program=(lw_x64_rec_program*)calloc(1u,sizeof(*program)); if(program==NULL){lw_session_free(session);lw_set_error(error,LW_STATUS_OUT_OF_MEMORY,"REC program allocation failed");return LW_X64_REC_COMPILE_OUT_OF_MEMORY;} program->model=model;program->cpu=lw_get_cpu_capabilities();program->model_signature=info.content_checksum;program->target_width=target_width;program->backend_layout=(uint8_t)strategy;program->value_count=info.tensor_count;program->direct_nhwc=(uint8_t)(strategy == LW_X64_REC_COMPILE_NHWC); + if (packed_source != NULL) { program->packed_source = packed_source; lw_x64_rec_program_retain((lw_x64_rec_program*)packed_source); } + if(!rec_backend_cpu_supported(program->cpu)){lw_session_free(session);lw_x64_rec_program_free(program);lw_set_error(error,LW_STATUS_UNSUPPORTED,"REC compiled backend requires a supported SIMD target");return LW_X64_REC_COMPILE_UNSUPPORTED;} program->ctc_fused=(allow_ctc_tail!=0u&&detect_ctc(model,session,&program->ctc))?1u:0u; limit=program->ctc_fused?info.node_count-3u:info.node_count; program->ops=(lw_x64_rec_op*)calloc(limit,sizeof(*program->ops));program->values=(lw_x64_rec_value*)calloc(program->value_count,sizeof(*program->values));if(program->ops==NULL||program->values==NULL){lw_session_free(session);lw_x64_rec_program_free(program);lw_set_error(error,LW_STATUS_OUT_OF_MEMORY,"REC program tables allocation failed");return LW_X64_REC_COMPILE_OUT_OF_MEMORY;} + for(i=0u;ivalue_count;++i){fill_value(session,i,&program->values[i],strategy);if((session->tensors[i].flags&LWM_V0_TENSOR_FLAG_INPUT)!=0u)program->input_value=i;} + lowered=lower_ops(model,session,program,limit,error); + if(lowered!=LW_X64_REC_COMPILE_OK){lw_session_free(session);lw_x64_rec_program_free(program);return lowered;} +#if defined(LW_EXPERIMENTAL_REC_FFN_TILING) + mark_ffn_pairs(program); +#endif + status=plan_arena_offsets(program,error); + if(status!=LW_STATUS_OK){lw_session_free(session);lw_x64_rec_program_free(program);return status==LW_STATUS_OUT_OF_MEMORY?LW_X64_REC_COMPILE_OUT_OF_MEMORY:LW_X64_REC_COMPILE_INVALID_GRAPH;} + reset_lowering(program,limit); + lowered=lower_ops(model,session,program,limit,error); + if(lowered!=LW_X64_REC_COMPILE_OK){lw_session_free(session);lw_x64_rec_program_free(program);return lowered;} +#if defined(LW_EXPERIMENTAL_REC_FFN_TILING) + mark_ffn_pairs(program); +#endif + program->semantic_consumed+=program->semantic_elided; if(program->ctc_fused){ + uint64_t ctc_count = 0u; + status=lw_x64_rec_ctc_prepare_shared(model,session,packed_source == NULL ? NULL : &packed_source->ctc,&program->ctc,error); + if(status!=LW_STATUS_OK){lw_session_free(session);lw_x64_rec_program_free(program);return status==LW_STATUS_OUT_OF_MEMORY?LW_X64_REC_COMPILE_OUT_OF_MEMORY:LW_X64_REC_COMPILE_INVALID_GRAPH;} + if (lw_packed_matmul_weight_count(program->ctc.inner, program->ctc.classes, &ctc_count)) { + if (program->ctc.packed_weights_borrowed != 0u) note_borrow(program, ctc_count * sizeof(float)); + else program->owned_constant_bytes += ctc_count * sizeof(float); + } + } + program->class_count=program->ctc.classes;program->time_steps=program->ctc.rows;program->output_value=info.output_count?lwm_read_u32(model->bytes+(size_t)model->output_offset):0u; if(program->arena_bytes==0u){lw_session_free(session);lw_x64_rec_program_free(program);lw_set_error(error,LW_STATUS_INVALID_SHAPE,"REC graph has no runtime arena");return LW_X64_REC_COMPILE_INVALID_GRAPH;} lw_session_free(session);lw_set_error(error,LW_STATUS_OK,"");*out_program=program;return LW_X64_REC_COMPILE_OK; +} + +lw_x64_rec_compile_result lw_x64_rec_backend_compile_input(const lw_model* model, uint32_t input_height, + uint32_t target_width, + lw_x64_rec_compile_strategy strategy, + uint32_t allow_ctc_tail, + lw_x64_rec_program** out_program, lw_error* error) { + return rec_backend_compile_input_impl(model, input_height, target_width, strategy, + allow_ctc_tail, NULL, out_program, error); +} + +/* Lower semantic nodes into the physical op table. Runs twice: the first pass + * discovers value lifetimes (producer/last_use/fusions/aliases) before arena + * offsets are planned; the second pass embeds the final offsets. */ +/* Walk the semantic producer chain of a tensor to determine the physical + * layout of its memory in the compiled program: conv-family ops produce + * physical NHWC, layout-transparent elementwise/unary ops inherit from + * their (non-constant) input, and everything else (transpose/matmul/ + * slice/reshape chains) produces canonical-order memory. */ +static int rec_value_is_nhwc_physical(const lw_model* model, const lw_session* session, + uint32_t tensor) { + uint32_t current = tensor; + uint32_t hops; + for (hops = 0u; hops < 4096u; ++hops) { + const uint8_t* node = NULL; + uint16_t semantic; + uint32_t j; + for (j = 0u; j < model->info.node_count; ++j) { + const uint8_t* n = node_bytes(model, j); + if (lwm_read_u32(n + 40u) == current) { node = n; break; } + } + if (node == NULL) return 1; /* graph input: preprocessing emits NHWC */ + semantic = lwm_read_u16(node); + switch (semantic) { + case LW_OP_ADD: case LW_OP_MUL: case LW_OP_DIV: case LW_OP_SUB: + case LW_OP_POW: case LW_OP_ERF: case LW_OP_HARD_SIGMOID: + case LW_OP_RELU: case LW_OP_SIGMOID: case LW_OP_SQRT: + case LW_OP_SOFTMAX: { + /* layout-transparent: inherit from the non-constant input */ + uint32_t next = lwm_read_u32(node + 8u); + if (lwm_read_u16(node + 2u) == 2u && + (session->tensors[next].flags & LWM_V0_TENSOR_FLAG_CONSTANT) != 0u) { + next = lwm_read_u32(node + 12u); + } + current = next; + break; + } + case LW_OP_CONV: case LW_OP_CONV_TRANSPOSE: + case LW_OP_BATCH_NORMALIZATION: case LW_OP_REDUCE_MEAN: + case LW_OP_AVERAGE_POOL: case LW_OP_MAX_POOL: + case LW_OP_CONCAT: case LW_OP_RESIZE: + return 1; + default: + return 0; + } + } + return 0; +} + +static lw_x64_rec_compile_result lower_ops(const lw_model* model, const lw_session* session, + lw_x64_rec_program* program, uint32_t limit, + lw_error* error) { + uint32_t i; + uint32_t physical = 0u; + lw_status status = LW_STATUS_OK; + for(i=0u;iops[physical]; + const lw_x64_rec_value* source = &program->values[match.inputs[0][0]]; + const lw_x64_rec_value* output = &program->values[match.outputs[4]]; + memset(op, 0, sizeof(*op)); + op->kind = LW_X64_REC_OP_GELU; + op->semantic_begin = i; + op->semantic_count = 5u; + op->data.unary.input_offset = source->offset; + op->data.unary.output_offset = output->offset; + op->data.unary.element_count = output->bytes / sizeof(float); + op->data.unary.rank = output->rank; + memcpy(op->data.unary.dimensions, output->dimensions, + sizeof(op->data.unary.dimensions)); + program->values[match.outputs[4]].producer = (int32_t)physical; + for (uint32_t fused_node = i; fused_node < i + 5u; ++fused_node) { + const uint8_t* fused_bytes = node_bytes(model, fused_node); + uint16_t fused_inputs = lwm_read_u16(fused_bytes + 2u); + for (uint16_t input_index = 0u; input_index < fused_inputs; ++input_index) { + uint32_t input = lwm_read_u32( + fused_bytes + 8u + (size_t)input_index * sizeof(uint32_t)); + if (program->values[input].last_use < (int32_t)physical) { + program->values[input].last_use = (int32_t)physical; + } + } + } + ++physical; + program->semantic_consumed += 5u; + i += 4u; + continue; + } + } + if (program->backend_layout == LW_X64_REC_BACKEND_NCHW && + (semantic == LW_OP_RESHAPE || semantic == LW_OP_SQUEEZE || + semantic == LW_OP_UNSQUEEZE)) { + uint32_t in = lwm_read_u32(node + 8u); + uint32_t out = lwm_read_u32(node + 40u); + program->values[out].alias = 1u; + program->values[out].alias_of = in; + program->values[out].offset = program->values[in].offset; + ++program->semantic_elided; + program->values[out].producer = (int32_t)i; + continue; + } + if(semantic==LW_OP_RESHAPE||semantic==LW_OP_SQUEEZE||semantic==LW_OP_UNSQUEEZE){uint32_t in=lwm_read_u32(node+8u),out=lwm_read_u32(node+40u); const lw_runtime_tensor* in_tensor=&session->tensors[in]; const lw_runtime_tensor* out_tensor=&session->tensors[out]; if(in_tensor->rank==3u && out_tensor->rank==4u && physicalops[physical++]; memset(repack,0,sizeof(*repack)); repack->kind=LW_X64_REC_OP_TRANSPOSE; repack->semantic_begin=i; repack->semantic_count=1u; repack->data.transpose.input_offset=program->values[in].offset; repack->data.transpose.output_offset=program->values[out].offset; repack->data.transpose.rank=4u; if(out_tensor->dimensions[1]==in_tensor->dimensions[1]){ repack->data.transpose.input_dimensions[0]=in_tensor->dimensions[0]; repack->data.transpose.input_dimensions[1]=in_tensor->dimensions[1]; repack->data.transpose.input_dimensions[2]=out_tensor->dimensions[2]; repack->data.transpose.input_dimensions[3]=out_tensor->dimensions[3]; repack->data.transpose.output_dimensions[0]=out_tensor->dimensions[0]; repack->data.transpose.output_dimensions[1]=out_tensor->dimensions[2]; repack->data.transpose.output_dimensions[2]=out_tensor->dimensions[3]; repack->data.transpose.output_dimensions[3]=out_tensor->dimensions[1]; repack->data.transpose.permutation[0]=0; repack->data.transpose.permutation[1]=2; repack->data.transpose.permutation[2]=3; repack->data.transpose.permutation[3]=1; }else{ /* Contiguous sequence-to-map repack whose permuted encoding is not * self-consistent (e.g. [1,W,C] -> [1,1,W,C]); both layouts hold the * same bytes, so emit the identity view and let the executor * degenerate to a plain copy. */ repack->data.transpose.input_dimensions[0]=out_tensor->dimensions[0]; repack->data.transpose.input_dimensions[1]=out_tensor->dimensions[1]; repack->data.transpose.input_dimensions[2]=out_tensor->dimensions[2]; repack->data.transpose.input_dimensions[3]=out_tensor->dimensions[3]; repack->data.transpose.output_dimensions[0]=out_tensor->dimensions[0]; repack->data.transpose.output_dimensions[1]=out_tensor->dimensions[1]; repack->data.transpose.output_dimensions[2]=out_tensor->dimensions[2]; repack->data.transpose.output_dimensions[3]=out_tensor->dimensions[3]; repack->data.transpose.permutation[0]=0; repack->data.transpose.permutation[1]=1; repack->data.transpose.permutation[2]=2; repack->data.transpose.permutation[3]=3; } program->values[out].producer=(int32_t)(physical-1u); if(program->values[in].last_use<(int32_t)(physical-1u))program->values[in].last_use=(int32_t)(physical-1u); program->semantic_consumed++; continue; } if(in_tensor->rank==4u && out_tensor->rank==3u && !rec_value_is_nhwc_physical(model,session,in)){ /* Canonical-order rank-4 input (attention chain): the reshape is a pure reinterpretation, elide it. */ program->values[out].alias=1u;program->values[out].alias_of=in;program->values[out].offset=program->values[in].offset;++program->semantic_elided;program->values[out].producer=(int32_t)i;continue; } if(in_tensor->rank==4u && out_tensor->rank==3u && physicalops[physical++]; memset(repack,0,sizeof(*repack)); repack->kind=LW_X64_REC_OP_TRANSPOSE; repack->semantic_begin=i; repack->semantic_count=1u; repack->data.transpose.input_offset=program->values[in].offset; repack->data.transpose.output_offset=program->values[out].offset; repack->data.transpose.rank=4u; repack->data.transpose.input_dimensions[0]=in_tensor->dimensions[0]; repack->data.transpose.input_dimensions[1]=in_tensor->dimensions[2]; repack->data.transpose.input_dimensions[2]=in_tensor->dimensions[3]; repack->data.transpose.input_dimensions[3]=in_tensor->dimensions[1]; repack->data.transpose.output_dimensions[0]=out_tensor->dimensions[0]; repack->data.transpose.output_dimensions[1]=in_tensor->dimensions[1]; repack->data.transpose.output_dimensions[2]=in_tensor->dimensions[2]; repack->data.transpose.output_dimensions[3]=in_tensor->dimensions[3]; repack->data.transpose.permutation[0]=0; repack->data.transpose.permutation[1]=3; repack->data.transpose.permutation[2]=1; repack->data.transpose.permutation[3]=2; program->values[out].producer=(int32_t)(physical-1u); if(program->values[in].last_use<(int32_t)(physical-1u))program->values[in].last_use=(int32_t)(physical-1u); program->semantic_consumed++; continue; } program->values[out].alias=1u;program->values[out].alias_of=in;program->values[out].offset=program->values[in].offset;++program->semantic_elided;program->values[out].producer=(int32_t)i;continue;} if(physical>=limit)return LW_X64_REC_COMPILE_INVALID_GRAPH; {lw_x64_rec_op* op=&program->ops[physical];memset(op,0,sizeof(*op));status=compile_node(model,session,program,i,op,error);if(status!=LW_STATUS_OK||op->kind==LW_X64_REC_OP_GENERIC_UNSUPPORTED){++program->unsupported_nodes;if (status == LW_STATUS_OK || error == NULL || error->message[0] == 0) { char message[128]; (void)snprintf(message,sizeof(message),"REC node %u (op %u) lowering failed (status %d, kind %u)",(unsigned)i,(unsigned)semantic,(int)status,(unsigned)op->kind); lw_set_error(error,status!=LW_STATUS_OK?status:LW_STATUS_UNSUPPORTED,message); }return LW_X64_REC_COMPILE_UNSUPPORTED;}op->semantic_begin=i;op->semantic_count=1u;if(program->backend_layout==LW_X64_REC_BACKEND_NHWC&&op->data.conv.scalar_fallback==0u&&(op->kind==LW_X64_REC_OP_POINTWISE||op->kind==LW_X64_REC_OP_DENSE||op->kind==LW_X64_REC_OP_DEPTHWISE)){uint32_t fold_span=fold_conv_epilogue(model,session,program,limit,i,op,physical);if(fold_span>1u){op->semantic_count=(uint16_t)fold_span;program->semantic_consumed+=fold_span;++physical;i+=fold_span-1u;continue;}}program->semantic_consumed++;program->values[lwm_read_u32(node+40u)].producer=(int32_t)physical;for(uint32_t j=0u;jvalues[in].last_use<(int32_t)physical)program->values[in].last_use=(int32_t)physical;}++physical;}} + program->op_count=physical; + return LW_X64_REC_COMPILE_OK; +} + +lw_x64_rec_compile_result lw_x64_rec_backend_compile_ex( + const lw_model* model, uint32_t target_width, + lw_x64_rec_compile_strategy strategy, + lw_x64_rec_program** out_program, lw_error* error) { +#if defined(__EMSCRIPTEN__) + if (strategy != LW_X64_REC_COMPILE_NHWC) { + if (out_program != NULL) *out_program = NULL; + lw_set_error(error, LW_STATUS_UNSUPPORTED, + "WASM compiled REC supports NHWC layout only"); + return LW_X64_REC_COMPILE_UNSUPPORTED; + } +#endif + return lw_x64_rec_backend_compile_input(model, 48u, target_width, strategy, 1u, + out_program, error); +} + +lw_x64_rec_compile_result lw_x64_rec_backend_compile( + const lw_model* model, uint32_t target_width, + lw_x64_rec_program** out_program, lw_error* error) { + return lw_x64_rec_backend_compile_input(model, 48u, target_width, + LW_X64_REC_COMPILE_NHWC, 1u, out_program, error); +} + +lw_x64_rec_compile_result lw_x64_rec_backend_compile_shared( + const lw_model* model, uint32_t target_width, const lw_x64_rec_program* packed_source, + lw_x64_rec_program** out_program, lw_error* error) { + if (packed_source == NULL) { + if (out_program != NULL) *out_program = NULL; + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "REC packed source is null"); + return LW_X64_REC_COMPILE_INVALID_GRAPH; + } + return rec_backend_compile_input_impl(model, 48u, target_width, + LW_X64_REC_COMPILE_NHWC, 1u, packed_source, + out_program, error); +} + +void lw_x64_rec_program_retain(lw_x64_rec_program* program) { + uint32_t current; + if (program == NULL) return; + current = lw_atomic_u32_load_acquire(&program->shared_refs); + while (current != UINT32_MAX && + !lw_atomic_u32_compare_exchange_acq_rel(&program->shared_refs, ¤t, + current + 1u)) {} +} + +void lw_x64_rec_program_free(lw_x64_rec_program* program) { + uint32_t current; + uint32_t i; + const lw_x64_rec_program* packed_source; + if (program == NULL) return; + current = lw_atomic_u32_load_acquire(&program->shared_refs); + while (current != 0u) { + if (lw_atomic_u32_compare_exchange_acq_rel(&program->shared_refs, ¤t, + current - 1u)) return; + } + packed_source = program->packed_source; + lw_x64_rec_ctc_free(&program->ctc); + for (i = 0u; i < program->packed_constant_count; ++i) free(program->constants[i].data); + free(program->constants); + free(program->ops); + free(program->values); + free(program); + lw_x64_rec_program_free((lw_x64_rec_program*)packed_source); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_execute.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_execute.c new file mode 100644 index 00000000..23b7199c --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_execute.c @@ -0,0 +1,1584 @@ +#include "x64_rec_backend_internal.h" +#include "ctc_projection_internal.h" +#include "rec_backend_kernels_internal.h" +#include "operator_internal.h" +#include "../ppocr/profile_internal.h" +#include "../simd/simd_kernels.h" + +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#include +static void* rec_aligned_alloc(size_t alignment, size_t size) { return _aligned_malloc(size, alignment); } +static void rec_aligned_free(void* p) { _aligned_free(p); } +#else +static void* rec_aligned_alloc(size_t alignment, size_t size) { + void* p = NULL; + return posix_memalign(&p, alignment, size) == 0 ? p : NULL; +} +static void rec_aligned_free(void* p) { free(p); } +#endif + +static float* offset_ptr(lw_x64_rec_instance* instance, uint64_t offset) { + if (instance == NULL || instance->arena == NULL || offset > SIZE_MAX) return NULL; + return (float*)(void*)(instance->arena + (size_t)offset); +} + +static lw_scalar_binary_op binary_operation(uint16_t operation) { + switch (operation) { + case LW_OP_ADD: return LW_SCALAR_BINARY_ADD; + case LW_OP_MUL: return LW_SCALAR_BINARY_MUL; + case LW_OP_DIV: return LW_SCALAR_BINARY_DIV; + case LW_OP_SUB: return LW_SCALAR_BINARY_SUB; + default: return LW_SCALAR_BINARY_POW; + } +} + +static void scalar_nhwc_conv(const lw_x64_rec_conv_op* conv, const float* input, + float* output) { + uint32_t oy, ox, oc, ky, kx, ic; + for (oy = 0u; oy < conv->output_height; ++oy) { + for (ox = 0u; ox < conv->output_width; ++ox) { + for (oc = 0u; oc < conv->output_channels; ++oc) { + float sum = conv->bias == NULL ? 0.0f : conv->bias[oc]; + for (ky = 0u; ky < conv->kernel_h; ++ky) { + int32_t iy = (int32_t)(oy * conv->stride_h + ky) - (int32_t)conv->pad_top; + if (iy < 0 || iy >= (int32_t)conv->input_height) continue; + for (kx = 0u; kx < conv->kernel_w; ++kx) { + int32_t ix = (int32_t)(ox * conv->stride_w + kx) - (int32_t)conv->pad_left; + if (ix < 0 || ix >= (int32_t)conv->input_width) continue; + for (ic = 0u; ic < conv->input_channels; ++ic) { + size_t input_index = ((size_t)iy * conv->input_width + (size_t)ix) * conv->input_channels + ic; + size_t weight_index; + if (conv->groups == conv->input_channels) { + if (ic != oc) continue; + weight_index = (((size_t)oc * conv->kernel_h + ky) * conv->kernel_w) + kx; + } else { + weight_index = ((((size_t)oc * conv->input_channels + ic) * conv->kernel_h + ky) * conv->kernel_w) + kx; + } + sum += input[input_index] * conv->original_weights[weight_index]; } + } + } + output[((size_t)oy * conv->output_width + ox) * conv->output_channels + oc] = sum; + } + } + } +} +static void scalar_nhwc_batch_norm(const lw_x64_rec_affine_op* affine, const float* input, float* output) { + uint32_t pixel; + for (pixel = 0u; pixel < affine->pixels; ++pixel) { + uint32_t channel; + for (channel = 0u; channel < affine->channels; ++channel) { + float factor = affine->scale[channel] / sqrtf(affine->variance[channel] + affine->epsilon); + size_t index = affine->channel_major + ? (size_t)channel * affine->pixels + pixel + : (size_t)pixel * affine->channels + channel; + float value = input[index]; + output[index] = (value - affine->mean[channel]) * factor + affine->bias[channel]; + } + } +} + +/* Intra-op sharding for 1x1 pointwise convs: pixels are independent and each + * output pixel accumulates over the input channels in the kernel's fixed + * order, so any contiguous pixel split is bit-identical to the serial run. */ +typedef struct lw_rec_pointwise_shard { + const float* input; + const float* packed_weights; + lw_nhwc_epilogue epilogue; + float* output; + uint32_t pixels; + uint32_t input_channels; + uint32_t output_channels; + uint16_t kernel; +} lw_rec_pointwise_shard; + +static void pointwise_run_range(const lw_rec_pointwise_shard* shard, + uint32_t pixel_begin, uint32_t pixel_count) { + lw_nhwc_epilogue ep = shard->epilogue; + const float* input = shard->input + (size_t)pixel_begin * shard->input_channels; + float* output = shard->output + (size_t)pixel_begin * shard->output_channels; + if (ep.residual != NULL) { + ep.residual += (size_t)pixel_begin * shard->output_channels; + } +#if defined(__EMSCRIPTEN__) + lw_rec_backend_kernels_current()->pointwise( + input, shard->packed_weights, &ep, output, pixel_count, + shard->input_channels, shard->output_channels); +#else + switch (shard->kernel) { + case LW_X64_REC_PW_4X16: + lw_avx2_fma_nhwc_pointwise_4x16_f32(input, shard->packed_weights, &ep, output, + pixel_count, shard->input_channels, + shard->output_channels); + break; + case LW_X64_REC_PW_3X32: + lw_avx2_fma_nhwc_pointwise_3x32_f32(input, shard->packed_weights, &ep, output, + pixel_count, shard->input_channels, + shard->output_channels); + break; + case LW_X64_REC_PW_2X32: + lw_avx2_fma_nhwc_pointwise_2x32_f32(input, shard->packed_weights, &ep, output, + pixel_count, shard->input_channels, + shard->output_channels); + break; + default: + lw_avx2_fma_nhwc_pointwise_f32(input, shard->packed_weights, &ep, output, + pixel_count, shard->input_channels, + shard->output_channels); + break; + } +#endif +} + +/* Shard begins use 12-pixel blocks: the kernel tile widths (2, 3, 4, 6) all + * divide 12, so every worker starts on a clean tile boundary. The final + * shard takes the arbitrary tail. */ +#define LW_REC_POINTWISE_SHARD_GRAIN 12u +static uint32_t pointwise_shard_begin(uint32_t pixels, uint32_t worker_count, + uint32_t worker_index) { + uint32_t blocks = (pixels + LW_REC_POINTWISE_SHARD_GRAIN - 1u) / + LW_REC_POINTWISE_SHARD_GRAIN; + uint32_t base = blocks / worker_count; + uint32_t remainder = blocks % worker_count; + uint32_t block = worker_index * base + (worker_index < remainder ? worker_index : remainder); + uint32_t begin = block * LW_REC_POINTWISE_SHARD_GRAIN; + return begin > pixels ? pixels : begin; +} + +static void pointwise_shard_entry(void* context_void, uint32_t worker_index, + uint32_t worker_count) { + const lw_rec_pointwise_shard* shard = (const lw_rec_pointwise_shard*)context_void; + uint32_t begin = pointwise_shard_begin(shard->pixels, worker_count, worker_index); + uint32_t end = pointwise_shard_begin(shard->pixels, worker_count, worker_index + 1u); + if (end > begin) { + pointwise_run_range(shard, begin, end - begin); + } +} + +/* Minimum MACs before worker fan-out pays for synchronization. */ +#define LW_REC_POINTWISE_SHARD_MIN_WORK (4ull * 1000ull * 1000ull) +static uint32_t pointwise_shard_workers(const lw_x64_rec_instance* instance, + uint32_t pixels, uint32_t input_channels, + uint32_t output_channels) { + uint64_t work; + uint32_t workers; + if (instance->thread_pool == NULL || instance->intra_op_workers <= 1u || + pixels < 2u * LW_REC_POINTWISE_SHARD_GRAIN) { + return 1u; + } + work = (uint64_t)pixels * input_channels * output_channels; + if (work < LW_REC_POINTWISE_SHARD_MIN_WORK) { + return 1u; + } + workers = instance->intra_op_workers; + if (workers > pixels / LW_REC_POINTWISE_SHARD_GRAIN) { + workers = pixels / LW_REC_POINTWISE_SHARD_GRAIN; + } + return workers <= 1u ? 1u : workers; +} + +/* Cache-local FFN scheduling. A tile boundary must divide every supported + * Pointwise row tile (2/3/4/6); channel accumulation and epilogues stay in the + * existing kernels. Reuse the planned intermediate slot: no new allocation. */ +#ifndef LW_REC_FFN_TILE_PIXELS +#define LW_REC_FFN_TILE_PIXELS 96u +#endif +typedef struct lw_rec_ffn_shard { + lw_rec_pointwise_shard expansion; + lw_rec_pointwise_shard projection; +} lw_rec_ffn_shard; + +static void ffn_shard_entry(void* context_void, uint32_t worker_index, + uint32_t worker_count) { + const lw_rec_ffn_shard* pair = (const lw_rec_ffn_shard*)context_void; + uint32_t begin = pointwise_shard_begin(pair->expansion.pixels, worker_count, worker_index); + uint32_t end = pointwise_shard_begin(pair->expansion.pixels, worker_count, worker_index + 1u); + while (begin < end) { + uint32_t count = end - begin; + if (count > LW_REC_FFN_TILE_PIXELS) count = LW_REC_FFN_TILE_PIXELS; + pointwise_run_range(&pair->expansion, begin, count); + pointwise_run_range(&pair->projection, begin, count); + begin += count; + } +} + +static int ffn_span_valid(uint64_t offset, uint64_t bytes, uint64_t capacity) { + return offset <= capacity && bytes <= capacity - offset; +} + +static int ffn_disjoint(uint64_t a, uint64_t a_bytes, uint64_t b, uint64_t b_bytes) { + return a <= b ? a_bytes <= b - a : b_bytes <= a - b; +} + +static void ffn_init_shard(lw_x64_rec_instance* instance, + const lw_x64_rec_conv_op* conv, uint32_t pixels, + lw_rec_pointwise_shard* shard) { + memset(shard, 0, sizeof(*shard)); + shard->input = offset_ptr(instance, conv->input_offset); + shard->output = offset_ptr(instance, conv->output_offset); + shard->packed_weights = conv->packed_weights; + shard->pixels = pixels; + shard->input_channels = conv->input_channels; + shard->output_channels = conv->output_channels; + shard->kernel = conv->pointwise_kernel; + shard->epilogue.bias = conv->bias; + shard->epilogue.post_bias = conv->post_bias; + shard->epilogue.activation = conv->activation; + if (conv->has_residual) + shard->epilogue.residual = offset_ptr(instance, conv->residual_offset); +} + +lw_status lw_x64_rec_instance_run_ffn_pair(lw_x64_rec_instance* instance, + uint32_t op_index, lw_error* error) { + const lw_x64_rec_program* program; + const lw_x64_rec_op* first; + const lw_x64_rec_op* second; + const lw_x64_rec_conv_op *a, *b; + uint64_t pixels64, input_bytes, middle_bytes, output_bytes; + uint32_t pixels, workers; + lw_rec_ffn_shard pair; + if (instance == NULL || instance->program == NULL || instance->arena == NULL || + op_index >= instance->program->op_count || + instance->program->op_count - op_index < 2u) goto invalid; + program = instance->program; + if (program->arena_bytes > SIZE_MAX) goto invalid; + first = &program->ops[op_index]; + second = &program->ops[op_index + 1u]; + a = &first->data.conv; + b = &second->data.conv; + if (!(first->flags & LW_X64_REC_OP_FFN_BEGIN) || + first->kind != LW_X64_REC_OP_POINTWISE || second->kind != LW_X64_REC_OP_POINTWISE || + a->scalar_fallback || b->scalar_fallback || a->has_residual || + a->kernel_h != 1u || a->kernel_w != 1u || b->kernel_h != 1u || b->kernel_w != 1u || + a->stride_h != 1u || a->stride_w != 1u || b->stride_h != 1u || b->stride_w != 1u || + a->groups != 1u || b->groups != 1u || + a->activation != LW_NHWC_ACT_GELU || b->activation != LW_NHWC_ACT_NONE || + a->packed_weights == NULL || b->packed_weights == NULL || + a->output_offset != b->input_offset || a->output_channels != b->input_channels || + a->input_channels == 0u || a->output_channels == 0u || b->output_channels == 0u || + a->input_height != b->input_height || a->input_width != b->input_width || + a->input_height != a->output_height || a->input_width != a->output_width || + b->input_height != b->output_height || b->input_width != b->output_width) goto invalid; + pixels64 = (uint64_t)a->input_height * a->input_width; + if (pixels64 == 0u || pixels64 > UINT32_MAX) goto invalid; + pixels = (uint32_t)pixels64; + if (pixels64 > UINT64_MAX / ((uint64_t)a->input_channels * sizeof(float)) || + pixels64 > UINT64_MAX / ((uint64_t)a->output_channels * sizeof(float)) || + pixels64 > UINT64_MAX / ((uint64_t)b->output_channels * sizeof(float))) goto invalid; + input_bytes = pixels64 * a->input_channels * sizeof(float); + middle_bytes = pixels64 * a->output_channels * sizeof(float); + output_bytes = pixels64 * b->output_channels * sizeof(float); + if (!ffn_span_valid(a->input_offset, input_bytes, program->arena_bytes) || + !ffn_span_valid(a->output_offset, middle_bytes, program->arena_bytes) || + !ffn_span_valid(b->output_offset, output_bytes, program->arena_bytes) || + !ffn_disjoint(a->input_offset, input_bytes, a->output_offset, middle_bytes) || + !ffn_disjoint(a->input_offset, input_bytes, b->output_offset, output_bytes) || + !ffn_disjoint(a->output_offset, middle_bytes, b->output_offset, output_bytes)) goto invalid; + if (b->has_residual && + (!ffn_span_valid(b->residual_offset, output_bytes, program->arena_bytes) || + !ffn_disjoint(b->residual_offset, output_bytes, a->output_offset, middle_bytes) || + !ffn_disjoint(b->residual_offset, output_bytes, b->output_offset, output_bytes))) goto invalid; + ffn_init_shard(instance, a, pixels, &pair.expansion); + ffn_init_shard(instance, b, pixels, &pair.projection); + workers = pointwise_shard_workers(instance, pixels, a->input_channels, a->output_channels); + if (workers > 1u) { + lw_thread_pool_run(instance->thread_pool, workers, ffn_shard_entry, &pair); + } else { + ffn_shard_entry(&pair, 0u, 1u); + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +invalid: + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "REC FFN pair/lifetime is invalid"); + return LW_STATUS_INVALID_ARGUMENT; +} + +/* Intra-op sharding for dense convs along output rows. The kernel accepts an + * output_row_offset and computes every output pixel with the same tap order + * as the serial run, so row splits are bit-identical. Each worker gets its + * own scratch slice: the per-op scratch geometry is row-count independent. */ +typedef struct lw_rec_dense_shard { + const float* input; + const float* packed_weights; + lw_nhwc_epilogue epilogue; + float* output; + lw_nhwc_dense_desc desc; + uint8_t* scratch_base; + uint64_t scratch_stride; + uint64_t scratch_bytes; + lw_status status[LW_PARALLEL_MAX_WORKERS]; +} lw_rec_dense_shard; + +static void dense_shard_entry(void* context_void, uint32_t worker_index, + uint32_t worker_count) { + lw_rec_dense_shard* shard = (lw_rec_dense_shard*)context_void; + const uint32_t full_rows = shard->desc.output_height + + shard->desc.output_row_offset; + uint32_t base = full_rows / worker_count; + uint32_t remainder = full_rows % worker_count; + uint32_t row_begin = worker_index * base + + (worker_index < remainder ? worker_index : remainder); + uint32_t row_end = row_begin + base + (worker_index < remainder ? 1u : 0u); + lw_nhwc_dense_desc desc; + lw_nhwc_epilogue ep; + float* output; + if (row_end <= row_begin) { + shard->status[worker_index] = LW_STATUS_OK; + return; + } + desc = shard->desc; + desc.output_height = row_end - row_begin; + desc.output_row_offset = row_begin; + ep = shard->epilogue; + output = shard->output + + (size_t)row_begin * desc.output_width * desc.output_channels; + if (ep.residual != NULL) { + ep.residual += (size_t)row_begin * desc.output_width * desc.output_channels; + } + shard->status[worker_index] = lw_avx2_fma_nhwc_dense_f32( + shard->input, shard->packed_weights, &ep, output, &desc, + shard->scratch_base + (size_t)worker_index * shard->scratch_stride, + shard->scratch_bytes); +} + +static uint32_t dense_shard_workers(const lw_x64_rec_instance* instance, + const lw_x64_rec_conv_op* conv) { + uint64_t work; + uint32_t workers; + uint32_t output_rows; + if (instance->thread_pool == NULL || instance->intra_op_workers <= 1u) { + return 1u; + } + output_rows = conv->output_height; + if (output_rows < 2u) { + return 1u; + } + work = (uint64_t)output_rows * conv->output_width * conv->output_channels * + conv->input_channels * conv->kernel_h * conv->kernel_w; + if (work < LW_REC_POINTWISE_SHARD_MIN_WORK) { + return 1u; + } + workers = instance->intra_op_workers; + if (workers > output_rows) { + workers = output_rows; + } + if (workers > LW_PARALLEL_MAX_WORKERS) { + workers = LW_PARALLEL_MAX_WORKERS; + } + return workers <= 1u ? 1u : workers; +} + +static uint8_t* dense_shard_scratch(lw_x64_rec_instance* instance, uint32_t workers, + uint64_t scratch_bytes) { + uint64_t needed; + if (scratch_bytes == 0u || workers <= 1u) { + return NULL; + } + needed = (uint64_t)workers * scratch_bytes; + if (instance->dense_shard_scratch_bytes < needed) { + uint8_t* grown; + rec_aligned_free(instance->dense_shard_scratch); + instance->dense_shard_scratch = NULL; + instance->dense_shard_scratch_bytes = 0u; + grown = (uint8_t*)rec_aligned_alloc(64u, (size_t)needed); + if (grown == NULL) { + return NULL; + } + instance->dense_shard_scratch = grown; + instance->dense_shard_scratch_bytes = needed; + } + return instance->dense_shard_scratch; +} + +/* Depthwise convs shard along output rows like dense ones; the kernel applies + * output_row_offset to the input window and needs no scratch. Every output + * pixel keeps the serial tap order, so splits are bit-identical. */ +typedef struct lw_rec_depthwise_shard { + const float* input; + const float* packed_weights; + const float* bias; + float* output; + lw_nhwc_depthwise_desc desc; + lw_status status[LW_PARALLEL_MAX_WORKERS]; +} lw_rec_depthwise_shard; + +static void depthwise_shard_entry(void* context_void, uint32_t worker_index, + uint32_t worker_count) { + lw_rec_depthwise_shard* shard = (lw_rec_depthwise_shard*)context_void; + const uint32_t full_rows = shard->desc.output_height + + shard->desc.output_row_offset; + uint32_t base = full_rows / worker_count; + uint32_t remainder = full_rows % worker_count; + uint32_t row_begin = worker_index * base + + (worker_index < remainder ? worker_index : remainder); + uint32_t row_end = row_begin + base + (worker_index < remainder ? 1u : 0u); + lw_nhwc_depthwise_desc desc; + float* output; + if (row_end <= row_begin) { + shard->status[worker_index] = LW_STATUS_OK; + return; + } + desc = shard->desc; + desc.output_height = row_end - row_begin; + desc.output_row_offset = row_begin; + output = shard->output + + (size_t)row_begin * desc.output_width * desc.channels; + shard->status[worker_index] = lw_avx2_fma_nhwc_depthwise_f32( + shard->input, shard->packed_weights, shard->bias, output, &desc); +} + +static uint32_t depthwise_shard_workers(const lw_x64_rec_instance* instance, + const lw_x64_rec_conv_op* conv) { + uint64_t work; + uint32_t workers; + uint32_t output_rows; + if (instance->thread_pool == NULL || instance->intra_op_workers <= 1u) { + return 1u; + } + output_rows = conv->output_height; + if (output_rows < 2u) { + return 1u; + } + work = (uint64_t)output_rows * conv->output_width * conv->input_channels * + conv->kernel_h * conv->kernel_w; + if (work < LW_REC_POINTWISE_SHARD_MIN_WORK) { + return 1u; + } + workers = instance->intra_op_workers; + if (workers > output_rows) { + workers = output_rows; + } + if (workers > LW_PARALLEL_MAX_WORKERS) { + workers = LW_PARALLEL_MAX_WORKERS; + } + return workers <= 1u ? 1u : workers; +} + +/* Matmul sharding. Two bit-identical modes: + * - mode 1: rank-2 shared-weight matmul, split over rows (4-row grain keeps + * the AVX2 row blocking; the kernel tail handles the remainder rows); + * - mode 2: attention-style rank-4 [1,B,M,K] x [1,B,K,N] (no broadcasting), + * split over B, each batch dispatched as a rank-2 matmul. + * Every output element accumulates over the inner axis in the serial order. */ +typedef struct lw_rec_matmul_shard { + const float* input; + const float* right; + float* output; + uint32_t batch; + uint32_t rows; + uint32_t inner; + uint32_t columns; + uint32_t mode; + lw_status status[LW_PARALLEL_MAX_WORKERS]; +} lw_rec_matmul_shard; + +static uint32_t matmul_shard_row_begin(uint32_t rows, uint32_t worker_count, + uint32_t worker_index) { + uint32_t blocks = (rows + 3u) / 4u; + uint32_t base = blocks / worker_count; + uint32_t remainder = blocks % worker_count; + uint32_t block = worker_index * base + (worker_index < remainder ? worker_index : remainder); + uint32_t begin = block * 4u; + return begin > rows ? rows : begin; +} + +static void matmul_shard_entry(void* context_void, uint32_t worker_index, + uint32_t worker_count) { + lw_rec_matmul_shard* shard = (lw_rec_matmul_shard*)context_void; + shard->status[worker_index] = LW_STATUS_OK; + if (shard->mode == 1u) { + uint32_t row_begin = matmul_shard_row_begin(shard->rows, worker_count, worker_index); + uint32_t row_end = matmul_shard_row_begin(shard->rows, worker_count, worker_index + 1u); + if (row_end <= row_begin) return; + shard->status[worker_index] = lw_matmul_shared_f32( + shard->input + (size_t)row_begin * shard->inner, shard->right, + shard->output + (size_t)row_begin * shard->columns, 1u, row_end - row_begin, + shard->inner, shard->columns); + } else { + /* Per-batch rank-2 view through the same scalar kernel the unsplit + * op would use, so every element keeps the exact accumulation order. */ + int32_t in_dims[2]; + int32_t w_dims[2]; + int32_t out_dims[2]; + uint32_t base = shard->batch / worker_count; + uint32_t remainder = shard->batch % worker_count; + uint32_t batch_begin = worker_index * base + + (worker_index < remainder ? worker_index : remainder); + uint32_t batch_end = batch_begin + base + (worker_index < remainder ? 1u : 0u); + uint32_t b; + in_dims[0] = (int32_t)shard->rows; in_dims[1] = (int32_t)shard->inner; + w_dims[0] = (int32_t)shard->inner; w_dims[1] = (int32_t)shard->columns; + out_dims[0] = (int32_t)shard->rows; out_dims[1] = (int32_t)shard->columns; + for (b = batch_begin; b < batch_end; ++b) { + lw_status status = lw_scalar_matmul_f32( + shard->input + (size_t)b * shard->rows * shard->inner, + shard->right + (size_t)b * shard->inner * shard->columns, + shard->output + (size_t)b * shard->rows * shard->columns, + 2u, in_dims, 2u, w_dims, 2u, out_dims); + if (status != LW_STATUS_OK) { + shard->status[worker_index] = status; + return; + } + } + } +} + +static uint32_t matmul_shard_workers(const lw_x64_rec_instance* instance, + const lw_x64_rec_matmul_op* matmul, + uint32_t* out_mode) { + uint64_t work = (uint64_t)matmul->batch * matmul->rows * matmul->inner * + matmul->columns; + uint32_t workers; + *out_mode = 0u; + if (instance->thread_pool == NULL || instance->intra_op_workers <= 1u || + work < LW_REC_POINTWISE_SHARD_MIN_WORK) { + return 1u; + } + workers = instance->intra_op_workers; + if (workers > LW_PARALLEL_MAX_WORKERS) { + workers = LW_PARALLEL_MAX_WORKERS; + } + if (matmul->weights_rank == 2u && matmul->batch == 1u && matmul->rows >= 8u) { + /* Rank-2: row split. */ + if (workers > matmul->rows / 4u) workers = matmul->rows / 4u; + *out_mode = 1u; + } else if (matmul->input_rank == 4u && matmul->weights_rank == 4u && + matmul->output_rank == 4u && matmul->batch > 1u && + matmul->input_dimensions[0] == 1 && matmul->weights_dimensions[0] == 1 && + matmul->output_dimensions[0] == 1 && + matmul->input_dimensions[1] == (int32_t)matmul->batch && + matmul->weights_dimensions[1] == (int32_t)matmul->batch && + matmul->output_dimensions[1] == (int32_t)matmul->batch) { + /* Attention-style batched matmul without broadcasting: batch split. */ + if (workers > matmul->batch) workers = matmul->batch; + *out_mode = 2u; + } else { + return 1u; + } + return workers <= 1u ? 1u : workers; +} + +static lw_status execute_op(lw_x64_rec_instance* instance, const lw_x64_rec_op* op, + lw_error* error) { + (void)error; + float* input; + float* output; + if (op == NULL) return LW_STATUS_INVALID_ARGUMENT; + switch (op->kind) { +#if !defined(__EMSCRIPTEN__) + case LW_X64_REC_OP_POINTWISE_NCHW: + case LW_X64_REC_OP_STEM_NCHW: + case LW_X64_REC_OP_DEPTHWISE_NCHW: { + int32_t input_dimensions[4] = { + 1, (int32_t)op->data.conv.input_channels, + (int32_t)op->data.conv.input_height, + (int32_t)op->data.conv.input_width + }; + int32_t output_dimensions[4] = { + 1, (int32_t)op->data.conv.output_channels, + (int32_t)op->data.conv.output_height, + (int32_t)op->data.conv.output_width + }; + input = offset_ptr(instance, op->data.conv.input_offset); + output = offset_ptr(instance, op->data.conv.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->kind == LW_X64_REC_OP_POINTWISE_NCHW) { + if (op->data.conv.nchw_pointwise_kernel == LW_X64_REC_NCHW_PW_FMA8) { + lw_avx2_fma_packed_conv1x1_8x8_f32( + input, op->data.conv.packed_weights, op->data.conv.bias, + output, input_dimensions, output_dimensions); + } else { + lw_avx2_fma_packed_conv1x1_f32( + input, op->data.conv.packed_weights, op->data.conv.bias, + output, input_dimensions, output_dimensions); + } + } else if (op->kind == LW_X64_REC_OP_STEM_NCHW) { + lw_avx2_fma_packed_conv3x3_stride2_pad1_f32( + input, op->data.conv.packed_weights, op->data.conv.bias, + output, input_dimensions, output_dimensions); + } else if (op->data.conv.kernel_h == 1u && op->data.conv.kernel_w == 5u) { + int32_t weight_dimensions[4] = { + (int32_t)op->data.conv.output_channels, 1, (int32_t)op->data.conv.weight_h, (int32_t)op->data.conv.weight_w + }; + int32_t kernel[2] = { 1, 5 }; + int32_t strides[2] = { (int32_t)op->data.conv.stride_h, (int32_t)op->data.conv.stride_w }; + int32_t dilations[2] = { 1, 1 }; + int32_t pads[4] = { (int32_t)op->data.conv.pad_top, (int32_t)op->data.conv.pad_left, + (int32_t)op->data.conv.pad_bottom, (int32_t)op->data.conv.pad_right }; + { + lw_status fallback_status = lw_scalar_conv2d_f32(input, op->data.conv.original_weights, op->data.conv.bias, + (op->data.conv.bias != NULL ? op->data.conv.output_channels : 0u), output, input_dimensions, + weight_dimensions, output_dimensions, kernel, strides, + dilations, pads, op->data.conv.groups); + if (fallback_status != LW_STATUS_OK) { lw_set_error(error, fallback_status, "NCHW depthwise fallback failed"); return fallback_status; } + } + } else if (op->data.conv.stride_h == 2u && op->data.conv.stride_w == 1u) { + lw_avx2_depthwise_conv3x3_stride2x1_pad1_f32( + input, op->data.conv.original_weights, op->data.conv.bias, + output, input_dimensions, output_dimensions); + } else { + lw_avx2_depthwise_conv3x3_unit_pad1_f32( + input, op->data.conv.original_weights, op->data.conv.bias, + output, input_dimensions); + } + return LW_STATUS_OK; + } +#endif + case LW_X64_REC_OP_POINTWISE: { + lw_nhwc_epilogue ep = { NULL, NULL, op->data.conv.activation, 0u, 0.0f, 0.0f, NULL }; + input = offset_ptr(instance, op->data.conv.input_offset); + output = offset_ptr(instance, op->data.conv.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + ep.bias = op->data.conv.bias; + ep.post_bias = op->data.conv.post_bias; + if (op->data.conv.has_residual != 0u) { + ep.residual = offset_ptr(instance, op->data.conv.residual_offset); + if (ep.residual == NULL) return LW_STATUS_INVALID_ARGUMENT; + } + { + uint32_t pixels = op->data.conv.input_height * op->data.conv.input_width; + uint32_t shard_workers = pointwise_shard_workers(instance, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); + if (shard_workers > 1u) { + lw_rec_pointwise_shard shard; + shard.input = input; + shard.packed_weights = op->data.conv.packed_weights; + shard.epilogue = ep; + shard.output = output; + shard.pixels = pixels; + shard.input_channels = op->data.conv.input_channels; + shard.output_channels = op->data.conv.output_channels; + shard.kernel = op->data.conv.pointwise_kernel; + lw_thread_pool_run(instance->thread_pool, shard_workers, + pointwise_shard_entry, &shard); + return LW_STATUS_OK; + } +#if defined(__EMSCRIPTEN__) + lw_rec_backend_kernels_current()->pointwise( + input, op->data.conv.packed_weights, &ep, output, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); +#else + switch (op->data.conv.pointwise_kernel) { + case LW_X64_REC_PW_4X16: + lw_avx2_fma_nhwc_pointwise_4x16_f32( + input, op->data.conv.packed_weights, &ep, output, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); + break; + case LW_X64_REC_PW_3X32: + lw_avx2_fma_nhwc_pointwise_3x32_f32( + input, op->data.conv.packed_weights, &ep, output, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); + break; + case LW_X64_REC_PW_2X32: + lw_avx2_fma_nhwc_pointwise_2x32_f32( + input, op->data.conv.packed_weights, &ep, output, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); + break; + default: + lw_avx2_fma_nhwc_pointwise_f32( + input, op->data.conv.packed_weights, &ep, output, pixels, + op->data.conv.input_channels, op->data.conv.output_channels); + break; + } +#endif + } + return LW_STATUS_OK; + } + case LW_X64_REC_OP_DENSE: { + lw_nhwc_dense_desc desc; + lw_nhwc_epilogue ep; + lw_status status; + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; + desc.input_channels = op->data.conv.input_channels; + desc.input_height = op->data.conv.input_height; + desc.input_width = op->data.conv.input_width; + desc.output_channels = op->data.conv.output_channels; + desc.output_height = op->data.conv.output_height; + desc.output_width = op->data.conv.output_width; + desc.kernel_h = op->data.conv.kernel_h; + desc.kernel_w = op->data.conv.kernel_w; + desc.stride_h = op->data.conv.stride_h; + desc.stride_w = op->data.conv.stride_w; + desc.pad_top = op->data.conv.pad_top; + desc.pad_left = op->data.conv.pad_left; + desc.pad_bottom = op->data.conv.pad_bottom; + desc.pad_right = op->data.conv.pad_right; + desc.dense_kc = op->data.conv.dense_kc; + memset(&ep, 0, sizeof(ep)); + ep.bias = op->data.conv.bias; + ep.activation = op->data.conv.activation; + ep.post_bias = op->data.conv.post_bias; + input = offset_ptr(instance, op->data.conv.input_offset); + output = offset_ptr(instance, op->data.conv.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.conv.has_residual != 0u) { + ep.residual = offset_ptr(instance, op->data.conv.residual_offset); + if (ep.residual == NULL) return LW_STATUS_INVALID_ARGUMENT; + } + { + uint32_t shard_workers = dense_shard_workers(instance, &op->data.conv); + if (shard_workers > 1u) { + uint8_t* shard_scratch = dense_shard_scratch(instance, shard_workers, + op->data.conv.scratch_bytes); + if (shard_scratch != NULL) { + lw_rec_dense_shard shard; + uint32_t wi; + shard.input = input; + shard.packed_weights = op->data.conv.packed_weights; + shard.epilogue = ep; + shard.output = output; + shard.desc = desc; + shard.scratch_base = shard_scratch; + shard.scratch_stride = op->data.conv.scratch_bytes; + shard.scratch_bytes = op->data.conv.scratch_bytes; + lw_thread_pool_run(instance->thread_pool, shard_workers, + dense_shard_entry, &shard); + status = LW_STATUS_OK; + for (wi = 0u; wi < shard_workers; ++wi) { + if (shard.status[wi] != LW_STATUS_OK) { + status = shard.status[wi]; + break; + } + } + if (status == LW_STATUS_OK) return LW_STATUS_OK; + if (ep.post_bias != NULL || ep.residual != NULL || + ep.activation != LW_NHWC_ACT_NONE) { + /* The scalar fallback cannot reproduce the fused epilogue. */ + return status; + } + scalar_nhwc_conv(&op->data.conv, input, output); + return LW_STATUS_OK; + } + } + } +#if defined(__EMSCRIPTEN__) + status = lw_rec_backend_kernels_current()->dense( +#else + status = lw_avx2_fma_nhwc_dense_f32( +#endif + input, op->data.conv.packed_weights, &ep, output, &desc, + instance->scratch, op->data.conv.scratch_bytes); + if (status == LW_STATUS_OK) return LW_STATUS_OK; + if (ep.post_bias != NULL || ep.residual != NULL || + ep.activation != LW_NHWC_ACT_NONE) { + /* The scalar fallback cannot reproduce the fused epilogue. */ + return status; + } + scalar_nhwc_conv(&op->data.conv, input, output); + return LW_STATUS_OK; + } + case LW_X64_REC_OP_DEPTHWISE: { + lw_nhwc_depthwise_desc desc; + lw_status status; + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; + desc.channels = op->data.conv.input_channels; + desc.input_height = op->data.conv.input_height; + desc.input_width = op->data.conv.input_width; + desc.output_height = op->data.conv.output_height; + desc.output_width = op->data.conv.output_width; + desc.kernel_h = op->data.conv.kernel_h; + desc.kernel_w = op->data.conv.kernel_w; + desc.stride_h = op->data.conv.stride_h; + desc.stride_w = op->data.conv.stride_w; + desc.pad_top = op->data.conv.pad_top; + desc.pad_left = op->data.conv.pad_left; + desc.pad_bottom = op->data.conv.pad_bottom; + desc.pad_right = op->data.conv.pad_right; + desc.post_bias = op->data.conv.post_bias; + input = offset_ptr(instance, op->data.conv.input_offset); + output = offset_ptr(instance, op->data.conv.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + { + uint32_t shard_workers = depthwise_shard_workers(instance, &op->data.conv); + if (shard_workers > 1u) { + lw_rec_depthwise_shard shard; + uint32_t wi; + shard.input = input; + shard.packed_weights = op->data.conv.packed_weights; + shard.bias = op->data.conv.bias; + shard.output = output; + shard.desc = desc; + lw_thread_pool_run(instance->thread_pool, shard_workers, + depthwise_shard_entry, &shard); + status = LW_STATUS_OK; + for (wi = 0u; wi < shard_workers; ++wi) { + if (shard.status[wi] != LW_STATUS_OK) { + status = shard.status[wi]; + break; + } + } + if (status == LW_STATUS_OK) return LW_STATUS_OK; + if (desc.post_bias != NULL) { + /* The scalar fallback cannot reproduce the fused epilogue. */ + return status; + } + scalar_nhwc_conv(&op->data.conv, input, output); + return LW_STATUS_OK; + } + } +#if defined(__EMSCRIPTEN__) + status = lw_rec_backend_kernels_current()->depthwise(input, op->data.conv.packed_weights, +#else + status = lw_avx2_fma_nhwc_depthwise_f32(input, op->data.conv.packed_weights, +#endif + op->data.conv.bias, output, &desc); + if (status == LW_STATUS_OK) return LW_STATUS_OK; + if (desc.post_bias != NULL) { + /* The scalar fallback cannot reproduce the fused epilogue. */ + return status; + } + scalar_nhwc_conv(&op->data.conv, input, output); + return LW_STATUS_OK; + } + case LW_X64_REC_OP_AFFINE: + input = offset_ptr(instance, op->data.affine.input_offset); output = offset_ptr(instance, op->data.affine.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.affine.channel_major && instance->program->backend_layout == LW_X64_REC_BACKEND_NCHW) { + lw_avx2_nchw_affine_f32(input, op->data.affine.mul, op->data.affine.add, + output, op->data.affine.channels, op->data.affine.pixels); + } else if (op->data.affine.channel_major) { + scalar_nhwc_batch_norm(&op->data.affine, input, output); + } else { +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_affine_nhwc_f32(input, op->data.affine.mul, op->data.affine.add, + output, op->data.affine.pixels, op->data.affine.channels); +#else + lw_avx2_nhwc_affine_f32(input, op->data.affine.mul, op->data.affine.add, + output, op->data.affine.pixels, op->data.affine.channels); +#endif + } + return LW_STATUS_OK; + case LW_X64_REC_OP_ADD: case LW_X64_REC_OP_MUL: case LW_X64_REC_OP_DIV: + case LW_X64_REC_OP_SUB: case LW_X64_REC_OP_POW: + input = op->data.binary.left_constant != NULL ? (float*)(uintptr_t)op->data.binary.left_constant : offset_ptr(instance, op->data.binary.left_offset); + output = offset_ptr(instance, op->data.binary.output_offset); + if (op->data.binary.channel_major_nchw != 0u && + instance->program->backend_layout == LW_X64_REC_BACKEND_NCHW && + (op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_LEFT_CHANNEL || + op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_RIGHT_CHANNEL)) { + float* full = op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_LEFT_CHANNEL + ? offset_ptr(instance, op->data.binary.right_offset) : input; + float* channel = op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_LEFT_CHANNEL + ? (op->data.binary.left_constant != NULL + ? (float*)(uintptr_t)op->data.binary.left_constant + : offset_ptr(instance, op->data.binary.left_offset)) + : (op->data.binary.right_constant != NULL + ? (float*)(uintptr_t)op->data.binary.right_constant + : offset_ptr(instance, op->data.binary.right_offset)); + if (full == NULL || channel == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + lw_avx2_binary_channel_nchw_f32( + binary_operation(op->data.binary.operation), full, channel, output, + op->data.binary.pixels, op->data.binary.channels, + op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_LEFT_CHANNEL); + return LW_STATUS_OK; + } + if (op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_LEFT_CHANNEL) { + float* full = offset_ptr(instance, op->data.binary.right_offset); + float* channel = op->data.binary.left_constant != NULL ? (float*)(uintptr_t)op->data.binary.left_constant : offset_ptr(instance, op->data.binary.left_offset); + if (full == NULL || channel == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_binary_channel_nhwc_f32(binary_operation(op->data.binary.operation), + full, channel, output, op->data.binary.pixels, op->data.binary.channels, 1); +#else + lw_avx2_binary_channel_f32(binary_operation(op->data.binary.operation), full, channel, output, op->data.binary.pixels, op->data.binary.channels, 1); +#endif + return LW_STATUS_OK; + } + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_RIGHT_SCALAR && op->data.binary.right_constant != NULL) { +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_binary_scalar_f32(binary_operation(op->data.binary.operation), input, + op->data.binary.right_constant[0], output, op->data.binary.element_count, 0); +#else + lw_avx2_binary_right_scalar_f32(binary_operation(op->data.binary.operation), input, + op->data.binary.right_constant[0], output, op->data.binary.element_count); +#endif + return LW_STATUS_OK; + } + if (op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_SAME) { + float* right = op->data.binary.right_constant != NULL ? (float*)(uintptr_t)op->data.binary.right_constant : offset_ptr(instance, op->data.binary.right_offset); + if (right == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.binary.mixed_nhwc != 0u) { + /* Cross-layout SAME: one side is canonical [N,C,H,W] order, + * the other physical NHWC. Gather the NHWC side through the + * index map so both operands align semantically. */ + uint32_t dn = (uint32_t)op->data.binary.dimensions[0]; + uint32_t dc = (uint32_t)op->data.binary.dimensions[1]; + uint32_t dh = (uint32_t)op->data.binary.dimensions[2]; + uint32_t dw = (uint32_t)op->data.binary.dimensions[3]; + uint32_t n, c, y, x; + for (n = 0u; n < dn; ++n) { + for (c = 0u; c < dc; ++c) { + for (y = 0u; y < dh; ++y) { + for (x = 0u; x < dw; ++x) { + uint64_t canonical_flat = ((uint64_t)(n * dc + c) * dh + y) * dw + x; + uint64_t nhwc_flat = ((uint64_t)(n * dh + y) * dw + x) * dc + c; + float lhs = op->data.binary.mixed_nhwc == 2u ? input[nhwc_flat] : input[canonical_flat]; + float rhs = op->data.binary.mixed_nhwc == 1u ? right[nhwc_flat] : right[canonical_flat]; + switch (op->data.binary.operation) { + case LW_OP_ADD: output[canonical_flat] = lhs + rhs; break; + case LW_OP_MUL: output[canonical_flat] = lhs * rhs; break; + case LW_OP_DIV: output[canonical_flat] = lhs / rhs; break; + case LW_OP_SUB: output[canonical_flat] = lhs - rhs; break; + default: output[canonical_flat] = powf(lhs, rhs); break; + } + } + } + } + } + return LW_STATUS_OK; + } +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_binary_contiguous_f32(binary_operation(op->data.binary.operation), + input, right, output, op->data.binary.element_count); +#else + lw_avx2_binary_contiguous_f32(binary_operation(op->data.binary.operation), input, right, output, op->data.binary.element_count); +#endif + return LW_STATUS_OK; + } + if (op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_RIGHT_CHANNEL) { + float* channel = op->data.binary.right_constant != NULL ? (float*)(uintptr_t)op->data.binary.right_constant : offset_ptr(instance, op->data.binary.right_offset); + if (channel == NULL) return LW_STATUS_INVALID_ARGUMENT; +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_binary_channel_nhwc_f32(binary_operation(op->data.binary.operation), + input, channel, output, op->data.binary.pixels, op->data.binary.channels, 0); +#else + lw_avx2_binary_channel_f32(binary_operation(op->data.binary.operation), input, channel, output, op->data.binary.pixels, op->data.binary.channels, 0); +#endif + return LW_STATUS_OK; + } + if (op->data.binary.broadcast_kind == LW_X64_REC_BROADCAST_RIGHT_PIXEL_SCALAR) { + const float* pixel_scalars = op->data.binary.right_constant != NULL + ? op->data.binary.right_constant + : offset_ptr(instance, op->data.binary.right_offset); + uint32_t pixel; + if (input == NULL || output == NULL || pixel_scalars == NULL) return LW_STATUS_INVALID_ARGUMENT; + for (pixel = 0u; pixel < op->data.binary.pixels; ++pixel) { +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_binary_scalar_f32(binary_operation(op->data.binary.operation), + input + (size_t)pixel * op->data.binary.channels, pixel_scalars[pixel], + output + (size_t)pixel * op->data.binary.channels, + op->data.binary.channels, 0); +#else + lw_avx2_binary_right_scalar_f32(binary_operation(op->data.binary.operation), + input + (size_t)pixel * op->data.binary.channels, pixel_scalars[pixel], + output + (size_t)pixel * op->data.binary.channels, op->data.binary.channels); +#endif + } + return LW_STATUS_OK; + } + if (op->data.binary.right_constant != NULL && op->data.binary.channels > 1u) { + uint64_t index; + const float* right = op->data.binary.right_constant; + for (index = 0u; index < op->data.binary.element_count; ++index) { + float lhs = input[index]; + float rhs = right[index % op->data.binary.channels]; + switch (op->data.binary.operation) { + case LW_OP_ADD: output[index] = lhs + rhs; break; + case LW_OP_MUL: output[index] = lhs * rhs; break; + case LW_OP_DIV: output[index] = lhs / rhs; break; + case LW_OP_SUB: output[index] = lhs - rhs; break; + default: output[index] = powf(lhs, rhs); break; + } + } + return LW_STATUS_OK; + } + return LW_STATUS_UNSUPPORTED; + case LW_X64_REC_OP_RELU: case LW_X64_REC_OP_ERF: case LW_X64_REC_OP_GELU: case LW_X64_REC_OP_HARD_SIGMOID: + input = offset_ptr(instance, op->data.unary.input_offset); output = offset_ptr(instance, op->data.unary.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->kind == LW_X64_REC_OP_RELU) { +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_relu_f32(input, output, op->data.unary.element_count); +#else + lw_avx2_relu_contiguous_f32(input, output, op->data.unary.element_count); +#endif + } + else if (op->kind == LW_X64_REC_OP_ERF) { +#if defined(__EMSCRIPTEN__) + lw_wasm128_erf_f32(input, output, op->data.unary.element_count); +#else + lw_avx2_erf_f32(input, output, op->data.unary.element_count); +#endif + } else if (op->kind == LW_X64_REC_OP_GELU) { +#if defined(__EMSCRIPTEN__) + lw_wasm128_gelu_f32(input, output, op->data.unary.element_count); +#else + lw_avx2_gelu_f32(input, output, op->data.unary.element_count); +#endif + } else { + lw_avx2_hard_sigmoid_contiguous_f32(input, output, + op->data.unary.element_count, op->data.unary.alpha, op->data.unary.beta); + } + return LW_STATUS_OK; + case LW_X64_REC_OP_SIGMOID: + /* Canonical scalar sigmoid: identical expf evaluation order, so + * results match the interpreter bit for bit. */ + input = offset_ptr(instance, op->data.unary.input_offset); output = offset_ptr(instance, op->data.unary.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + return lw_scalar_sigmoid_f32(input, output, op->data.unary.element_count); + case LW_X64_REC_OP_SQRT: + input = offset_ptr(instance, op->data.unary.input_offset); output = offset_ptr(instance, op->data.unary.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + return lw_scalar_sqrt_f32(input, output, op->data.unary.element_count); + case LW_X64_REC_OP_SLICE: + input = offset_ptr(instance, op->data.slice.input_offset); output = offset_ptr(instance, op->data.slice.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + return lw_scalar_slice_f32(input, output, op->data.slice.rank, + op->data.slice.input_dimensions, op->data.slice.output_dimensions, + op->data.slice.slice_count, op->data.slice.starts, op->data.slice.ends, + op->data.slice.axes, op->data.slice.steps); + case LW_X64_REC_OP_CONCAT: { + /* Channel-axis concat over NHWC rank-4 maps: per-pixel + * concatenated channel copies (pure data movement). */ + const float* concat_inputs[LWM_V0_MAX_NODE_INPUTS]; + uint32_t concat_slot; + uint64_t concat_pixel; + output = offset_ptr(instance, op->data.concat.output_offset); + if (output == NULL) return LW_STATUS_INVALID_ARGUMENT; + for (concat_slot = 0u; concat_slot < op->data.concat.input_count; ++concat_slot) { + concat_inputs[concat_slot] = offset_ptr(instance, op->data.concat.input_offsets[concat_slot]); + if (concat_inputs[concat_slot] == NULL) return LW_STATUS_INVALID_ARGUMENT; + } + for (concat_pixel = 0u; concat_pixel < op->data.concat.pixels; ++concat_pixel) { + float* destination = output + (size_t)concat_pixel * op->data.concat.output_channels; + for (concat_slot = 0u; concat_slot < op->data.concat.input_count; ++concat_slot) { + uint32_t concat_channels = op->data.concat.input_channels[concat_slot]; + memcpy(destination, concat_inputs[concat_slot] + (size_t)concat_pixel * concat_channels, + (size_t)concat_channels * sizeof(float)); + destination += concat_channels; + } + } + return LW_STATUS_OK; + } + case LW_X64_REC_OP_SOFTMAX: + input = offset_ptr(instance, op->data.softmax.input_offset); output = offset_ptr(instance, op->data.softmax.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + return lw_scalar_softmax_f32(input, output, op->data.softmax.rank, + op->data.softmax.dimensions, op->data.softmax.axis); + case LW_X64_REC_OP_REDUCE_MEAN: + input = offset_ptr(instance, op->data.reduce.input_offset); output = offset_ptr(instance, op->data.reduce.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.reduce.general != 0u) { + /* Not the global-pool pattern: run the canonical scalar + * reduce with the original axes, bit-identical to the + * interpreter (e.g. Small's rank-3 last-axis mean). */ + return lw_scalar_reduce_mean_f32(input, output, op->data.reduce.input_rank, + op->data.reduce.input_dimensions, op->data.reduce.axes_count, + op->data.reduce.axes, op->data.reduce.keep_dimensions, + op->data.reduce.no_op_with_empty_axes, op->data.reduce.output_rank, + op->data.reduce.output_dimensions); + } + if (instance->program->backend_layout == LW_X64_REC_BACKEND_NCHW) { + for (uint32_t channel = 0u; channel < op->data.reduce.channels; ++channel) { + const float* source = input + (size_t)channel * op->data.reduce.height * + op->data.reduce.width; + float sum = 0.0f; + for (uint32_t spatial = 0u; spatial < op->data.reduce.height * + op->data.reduce.width; ++spatial) { + sum += source[spatial]; + } + output[channel] = sum / (float)(op->data.reduce.height * op->data.reduce.width); + } + } else { +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_reduce_mean_hw_f32(input, output, op->data.reduce.batch, + op->data.reduce.height, op->data.reduce.width, op->data.reduce.channels); +#else + lw_avx2_nhwc_reduce_mean_hw_f32(input, output, op->data.reduce.batch, + op->data.reduce.height, op->data.reduce.width, + op->data.reduce.channels); +#endif + } + return LW_STATUS_OK; + case LW_X64_REC_OP_AVG_POOL: case LW_X64_REC_OP_MAX_POOL: + input = offset_ptr(instance, op->data.pool.input_offset); output = offset_ptr(instance, op->data.pool.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (instance->program->backend_layout == LW_X64_REC_BACKEND_NCHW) { + return op->kind == LW_X64_REC_OP_MAX_POOL + ? lw_scalar_max_pool2d_f32(input, output, op->data.pool.input_dimensions, + op->data.pool.output_dimensions, op->data.pool.kernel, + op->data.pool.strides, op->data.pool.pads, 0u) + : lw_scalar_average_pool2d_f32(input, output, op->data.pool.input_dimensions, + op->data.pool.output_dimensions, op->data.pool.kernel, + op->data.pool.strides, op->data.pool.pads, 0u, + op->data.pool.count_include_pad); + } +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_pool_nhwc_f32(input, output, +#else + lw_avx2_nhwc_pool_f32(input, output, +#endif + (uint32_t)op->data.pool.input_dimensions[0], + (uint32_t)op->data.pool.input_dimensions[1], + (uint32_t)op->data.pool.input_dimensions[2], + (uint32_t)op->data.pool.output_dimensions[1], + (uint32_t)op->data.pool.output_dimensions[2], + (uint32_t)op->data.pool.input_dimensions[3], + (uint32_t)op->data.pool.kernel[0], (uint32_t)op->data.pool.kernel[1], + (uint32_t)op->data.pool.strides[0], (uint32_t)op->data.pool.strides[1], + (uint32_t)op->data.pool.pads[0], (uint32_t)op->data.pool.pads[1], + op->data.pool.count_include_pad, op->data.pool.is_max); + return LW_STATUS_OK; + case LW_X64_REC_OP_TRANSPOSE: { + uint32_t rank = op->data.transpose.rank; + uint32_t axis; + uint32_t nontrivial[2] = { 0u, 0u }; + uint32_t nontrivial_count = 0u; + uint64_t strides[4]; + uint64_t stride = 1u; + int handled = 0; + input = offset_ptr(instance, op->data.transpose.input_offset); output = offset_ptr(instance, op->data.transpose.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + /* Effective-2D detection: when at most two output axes have extent > 1 + * the transpose degenerates to a matrix transpose (or a plain copy), + * which the blocked AVX2 kernel handles far faster than the generic + * per-element scalar walk. Pure data movement, bit-identical. */ + if (rank <= 4u) { + for (axis = rank; axis > 0u; --axis) { + strides[axis - 1u] = stride; + stride *= (uint64_t)(uint32_t)op->data.transpose.input_dimensions[axis - 1u]; + } + for (axis = 0u; axis < rank; ++axis) { + int32_t source_axis = op->data.transpose.permutation[axis]; + if (source_axis < 0 || (uint32_t)source_axis >= rank || + op->data.transpose.output_dimensions[axis] != + op->data.transpose.input_dimensions[source_axis]) { + nontrivial_count = UINT32_MAX; + break; + } + if (op->data.transpose.output_dimensions[axis] > 1) { + if (nontrivial_count >= 2u) { + nontrivial_count = UINT32_MAX; + break; + } + nontrivial[nontrivial_count++] = axis; + } + } + if (nontrivial_count == 2u) { + uint32_t a = nontrivial[0]; + uint32_t b = nontrivial[1]; + uint64_t rows = (uint64_t)(uint32_t)op->data.transpose.output_dimensions[a]; + uint64_t cols = (uint64_t)(uint32_t)op->data.transpose.output_dimensions[b]; + uint64_t stride_a = strides[(uint32_t)op->data.transpose.permutation[a]]; + uint64_t stride_b = strides[(uint32_t)op->data.transpose.permutation[b]]; + if (stride_a == 1u && stride_b == rows && rows <= UINT32_MAX && + cols <= UINT32_MAX) { + /* out[i][j] = in[j][i]: true matrix transpose. */ + lw_avx2_transpose_2d_f32(input, output, (uint32_t)cols, + (uint32_t)rows); + handled = 1; + } else if (stride_a == cols && stride_b == 1u) { + memcpy(output, input, (size_t)(rows * cols) * sizeof(float)); + handled = 1; + } + } else if (nontrivial_count <= 1u) { + uint64_t elements = stride; + memcpy(output, input, (size_t)elements * sizeof(float)); + handled = 1; + } + } + if (handled != 0) return LW_STATUS_OK; + return lw_scalar_transpose_f32(input, output, op->data.transpose.rank, op->data.transpose.input_dimensions, op->data.transpose.rank, op->data.transpose.permutation, op->data.transpose.output_dimensions); + } + case LW_X64_REC_OP_MATMUL: { + const float* right; + input = offset_ptr(instance, op->data.matmul.input_offset); output = offset_ptr(instance, op->data.matmul.output_offset); + if (input == NULL || output == NULL) return LW_STATUS_INVALID_ARGUMENT; + if (op->data.matmul.packed_weights != NULL) { +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) + lw_wasm128_packed_matmul_shared_f32(input, op->data.matmul.packed_weights, + output, op->data.matmul.batch, op->data.matmul.rows, + op->data.matmul.inner, op->data.matmul.columns); +#else + lw_avx2_packed_matmul_shared_f32(input, op->data.matmul.packed_weights, output, op->data.matmul.batch, op->data.matmul.rows, op->data.matmul.inner, op->data.matmul.columns); +#endif + return LW_STATUS_OK; + } + right = op->data.matmul.weights != NULL ? op->data.matmul.weights : (const float*)offset_ptr(instance, op->data.matmul.weights_offset); + if (right == NULL) return LW_STATUS_INVALID_ARGUMENT; + { + uint32_t shard_mode = 0u; + uint32_t shard_workers = matmul_shard_workers(instance, &op->data.matmul, + &shard_mode); + if (shard_workers > 1u) { + lw_rec_matmul_shard shard; + uint32_t wi; + lw_status status = LW_STATUS_OK; + shard.input = input; + shard.right = right; + shard.output = output; + shard.batch = op->data.matmul.batch; + shard.rows = op->data.matmul.rows; + shard.inner = op->data.matmul.inner; + shard.columns = op->data.matmul.columns; + shard.mode = shard_mode; + lw_thread_pool_run(instance->thread_pool, shard_workers, + matmul_shard_entry, &shard); + for (wi = 0u; wi < shard_workers; ++wi) { + if (shard.status[wi] != LW_STATUS_OK) { + status = shard.status[wi]; + break; + } + } + return status; + } + } + if (op->data.matmul.weights_rank == 2u) { + return lw_matmul_shared_f32(input, right, output, op->data.matmul.batch, op->data.matmul.rows, op->data.matmul.inner, op->data.matmul.columns); + } + return lw_scalar_matmul_f32(input, right, output, op->data.matmul.input_rank, op->data.matmul.input_dimensions, op->data.matmul.weights_rank, op->data.matmul.weights_dimensions, op->data.matmul.output_rank, op->data.matmul.output_dimensions); } + default: return LW_STATUS_UNSUPPORTED; + } +} + +void lw_x64_rec_instance_set_thread_pool(lw_x64_rec_instance* instance, + lw_thread_pool* pool, uint32_t workers) { + if (instance == NULL) return; + if (pool == NULL || workers <= 1u) { pool = NULL; workers = 0u; } + instance->thread_pool = pool; + instance->intra_op_workers = workers; +} + +lw_status lw_x64_rec_instance_create(const lw_x64_rec_program* program, lw_x64_rec_instance** out, lw_error* error) { + lw_x64_rec_instance* instance; + if (out == NULL || program == NULL) { lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "REC instance requires a compiled program"); return LW_STATUS_INVALID_ARGUMENT; } + *out = NULL; instance = (lw_x64_rec_instance*)calloc(1u, sizeof(*instance)); + if (instance == NULL) { lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "REC instance allocation failed"); return LW_STATUS_OUT_OF_MEMORY; } + instance->program = program; + instance->owns_workspace = 1u; + if (program->arena_bytes > 0u) instance->arena = (uint8_t*)rec_aligned_alloc(64u, (size_t)program->arena_bytes); + if (program->scratch_bytes > 0u) instance->scratch = (uint8_t*)rec_aligned_alloc(64u, (size_t)program->scratch_bytes); + if ((program->arena_bytes > 0u && instance->arena == NULL) || (program->scratch_bytes > 0u && instance->scratch == NULL)) { lw_x64_rec_instance_free(instance); lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "REC instance workspace allocation failed"); return LW_STATUS_OUT_OF_MEMORY; } + if (program->ctc.enabled) { uint64_t rows = program->ctc.rows; if (rows == 0u || rows > SIZE_MAX / sizeof(float)) { lw_x64_rec_instance_free(instance); return LW_STATUS_INVALID_SHAPE; } instance->ctc_scores = (float*)rec_aligned_alloc(64u, (size_t)(rows * sizeof(float))); instance->best_indices = (uint32_t*)rec_aligned_alloc(64u, (size_t)(rows * sizeof(uint32_t))); instance->best_probabilities = (float*)rec_aligned_alloc(64u, (size_t)(rows * sizeof(float))); if (instance->ctc_scores == NULL || instance->best_indices == NULL || instance->best_probabilities == NULL) { lw_x64_rec_instance_free(instance); lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "REC CTC workspace allocation failed"); return LW_STATUS_OUT_OF_MEMORY; } } + *out = instance; lw_set_error(error, LW_STATUS_OK, ""); return LW_STATUS_OK; +} + +lw_status lw_x64_rec_instance_create_borrowed( + const lw_x64_rec_program* program, + uint8_t* arena, uint64_t arena_bytes, + uint8_t* scratch, uint64_t scratch_bytes, + float* ctc_scores, uint32_t* best_indices, float* best_probabilities, + uint64_t ctc_rows, lw_x64_rec_instance** out, lw_error* error) { + lw_x64_rec_instance* instance; + if (out == NULL || program == NULL) { lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "REC instance requires a compiled program"); return LW_STATUS_INVALID_ARGUMENT; } + if ((program->arena_bytes > 0u && (arena == NULL || arena_bytes < program->arena_bytes)) || + (program->scratch_bytes > 0u && (scratch == NULL || scratch_bytes < program->scratch_bytes)) || + (program->ctc.enabled && (ctc_rows < program->ctc.rows || ctc_scores == NULL || + best_indices == NULL || best_probabilities == NULL))) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "REC borrowed workspace is smaller than the program requires"); + return LW_STATUS_INVALID_ARGUMENT; + } + *out = NULL; instance = (lw_x64_rec_instance*)calloc(1u, sizeof(*instance)); + if (instance == NULL) { lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "REC instance allocation failed"); return LW_STATUS_OUT_OF_MEMORY; } + instance->program = program; + instance->arena = arena; + instance->scratch = scratch; + instance->ctc_scores = ctc_scores; + instance->best_indices = best_indices; + instance->best_probabilities = best_probabilities; + instance->owns_workspace = 0u; + *out = instance; lw_set_error(error, LW_STATUS_OK, ""); return LW_STATUS_OK; +} + +void lw_x64_rec_instance_free(lw_x64_rec_instance* instance) { if (instance == NULL) return; rec_aligned_free(instance->dense_shard_scratch); if (instance->owns_workspace != 0u) { rec_aligned_free(instance->ctc_scores); rec_aligned_free(instance->best_indices); rec_aligned_free(instance->best_probabilities); rec_aligned_free(instance->scratch); rec_aligned_free(instance->arena); } free(instance); } + +float* lw_x64_rec_instance_input(lw_x64_rec_instance* instance, uint64_t* element_count) { const lw_x64_rec_value* value; if (element_count != NULL) *element_count = 0u; if (instance == NULL || instance->program == NULL || instance->program->input_value >= instance->program->value_count) return NULL; value=&instance->program->values[instance->program->input_value]; if (element_count != NULL) *element_count=value->bytes/sizeof(float); return offset_ptr(instance,value->offset); } + +/* Env-gated (LW_X64_REC_PROFILE=1) per-kind op timing for the compiled + * backend. Zero overhead when disabled: one cached getenv per process. */ +#if defined(_MSC_VER) +#pragma warning(disable : 4996) +#endif +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +static uint64_t rec_profile_now_ns(void) { + static LARGE_INTEGER freq; + LARGE_INTEGER now; + if (freq.QuadPart == 0) QueryPerformanceFrequency(&freq); + QueryPerformanceCounter(&now); + return (uint64_t)((now.QuadPart * 1000000000ull) / (uint64_t)freq.QuadPart); +} +#else +#include +static uint64_t rec_profile_now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} +#endif + +static int rec_profile_enabled(void) { + static int cached = -1; + if (cached < 0) cached = lw_profile_env_present("LW_X64_REC_PROFILE") ? 1 : 0; + return cached; +} + +#if defined(__EMSCRIPTEN__) +static int rec_op_profile_enabled(void) { + static int cached = -1; + if (cached < 0) cached = lw_profile_env_is_one("LW_WASM_REC_OP_PROFILE") ? 1 : 0; + return cached; +} + +static const char* rec_op_kind_name(uint16_t kind) { + switch (kind) { + case LW_X64_REC_OP_POINTWISE: return "pointwise"; + case LW_X64_REC_OP_DENSE: return "dense"; + case LW_X64_REC_OP_DEPTHWISE: return "depthwise"; + case LW_X64_REC_OP_AFFINE: return "affine"; + case LW_X64_REC_OP_ADD: return "add"; + case LW_X64_REC_OP_MUL: return "mul"; + case LW_X64_REC_OP_DIV: return "div"; + case LW_X64_REC_OP_SUB: return "sub"; + case LW_X64_REC_OP_POW: return "pow"; + case LW_X64_REC_OP_RELU: return "relu"; + case LW_X64_REC_OP_ERF: return "erf"; + case LW_X64_REC_OP_GELU: return "gelu"; + case LW_X64_REC_OP_HARD_SIGMOID: return "hard_sigmoid"; + case LW_X64_REC_OP_SIGMOID: return "sigmoid"; + case LW_X64_REC_OP_SQRT: return "sqrt"; + case LW_X64_REC_OP_REDUCE_MEAN: return "reduce_mean"; + case LW_X64_REC_OP_AVG_POOL: return "avg_pool"; + case LW_X64_REC_OP_MAX_POOL: return "max_pool"; + case LW_X64_REC_OP_TRANSPOSE: return "transpose"; + case LW_X64_REC_OP_MATMUL: return "matmul"; + case LW_X64_REC_OP_CONCAT: return "concat"; + case LW_X64_REC_OP_RESIZE: return "resize"; + case LW_X64_REC_OP_SOFTMAX: return "softmax"; + case LW_X64_REC_OP_SLICE: return "slice"; + default: return "other"; + } +} + +static void rec_op_profile_record(const lw_x64_rec_program* program, uint32_t index, + uint64_t invocation, uint64_t elapsed, + lw_status status) { + const lw_x64_rec_op* op = &program->ops[index]; + fprintf(stderr, "WASM_REC_OP width=%u invocation=%llu index=%u semantic=%u " + "span=%u kind=%s elapsed=%.6f status=%d", + program->target_width, (unsigned long long)invocation, index, + op->semantic_begin, (unsigned)op->semantic_count, rec_op_kind_name(op->kind), + elapsed / 1e6, (int)status); + if (op->kind == LW_X64_REC_OP_POINTWISE || op->kind == LW_X64_REC_OP_DENSE || + op->kind == LW_X64_REC_OP_DEPTHWISE) { + const lw_x64_rec_conv_op* conv = &op->data.conv; + fprintf(stderr, " ic=%u oc=%u ih=%u iw=%u oh=%u ow=%u kh=%u kw=%u " + "sh=%u sw=%u groups=%u act=%u residual=%u post_bias=%u input=%llu output=%llu", + conv->input_channels, conv->output_channels, conv->input_height, conv->input_width, + conv->output_height, conv->output_width, conv->kernel_h, conv->kernel_w, + conv->stride_h, conv->stride_w, conv->groups, (unsigned)conv->activation, + (unsigned)conv->has_residual, conv->post_bias != NULL ? 1u : 0u, + (unsigned long long)conv->input_offset, (unsigned long long)conv->output_offset); + } + fputc('\n', stderr); +} +#endif + +static uint64_t* rec_profile_slot(lw_x64_rec_profile* profile, uint16_t kind) { + switch (kind) { + case LW_X64_REC_OP_POINTWISE: return &profile->pointwise_ns; + case LW_X64_REC_OP_DENSE: return &profile->dense_ns; + case LW_X64_REC_OP_DEPTHWISE: return &profile->depthwise_ns; + case LW_X64_REC_OP_ADD: case LW_X64_REC_OP_MUL: case LW_X64_REC_OP_DIV: + case LW_X64_REC_OP_SUB: case LW_X64_REC_OP_POW: return &profile->binary_ns; + case LW_X64_REC_OP_RELU: case LW_X64_REC_OP_ERF: case LW_X64_REC_OP_GELU: + case LW_X64_REC_OP_HARD_SIGMOID: case LW_X64_REC_OP_SIGMOID: + case LW_X64_REC_OP_SQRT: return &profile->unary_ns; + case LW_X64_REC_OP_REDUCE_MEAN: return &profile->reduce_ns; + case LW_X64_REC_OP_AVG_POOL: case LW_X64_REC_OP_MAX_POOL: return &profile->pool_ns; + case LW_X64_REC_OP_TRANSPOSE: return &profile->transpose_ns; + case LW_X64_REC_OP_MATMUL: return &profile->matmul_ns; + default: return NULL; + } +} + +enum rec_pw_epi_kind { + REC_PW_PLAIN = 0, + REC_PW_BIAS, + REC_PW_HARDSWISH, + REC_PW_RESIDUAL, + REC_PW_RESIDUAL_GELU, + REC_PW_POSTBIAS_GELU, + REC_PW_OTHER, + REC_PW_EPI_COUNT +}; + +static uint32_t classify_rec_pointwise(const lw_x64_rec_conv_op* conv) { + if (conv->post_bias == NULL && conv->has_residual == 0u) { + if (conv->activation == LW_NHWC_ACT_NONE) + return conv->bias == NULL ? REC_PW_PLAIN : REC_PW_BIAS; + if (conv->activation == LW_NHWC_ACT_HARDSWISH) return REC_PW_HARDSWISH; + } + if (conv->post_bias == NULL && conv->has_residual != 0u && + conv->activation == LW_NHWC_ACT_NONE) return REC_PW_RESIDUAL; + if (conv->has_residual != 0u && conv->activation == LW_NHWC_ACT_GELU) + return REC_PW_RESIDUAL_GELU; + if (conv->post_bias != NULL && conv->activation == LW_NHWC_ACT_GELU) + return REC_PW_POSTBIAS_GELU; + return REC_PW_OTHER; +} + +static void record_rec_pointwise_epilogue(lw_x64_rec_profile* profile, + const lw_x64_rec_op* op, + uint64_t elapsed) { + const lw_x64_rec_conv_op* conv = &op->data.conv; + uint32_t kind = classify_rec_pointwise(conv); + uint64_t pixels = (uint64_t)conv->input_height * conv->input_width; + profile->pw_epi_ns[kind] += elapsed; + ++profile->pw_epi_calls[kind]; + profile->pw_epi_macs[kind] += pixels * conv->input_channels * conv->output_channels; +} + +static lw_status run_backbone_ops(lw_x64_rec_instance* instance, lw_error* error) { + uint32_t i; + lw_status status; + int profiling; +#if defined(__EMSCRIPTEN__) + static uint64_t next_invocation = 0u; /* Single-threaded diagnostic runs only. */ + uint64_t invocation = 0u; + int detailed = 0; +#endif + if (instance == NULL || instance->program == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "REC instance is invalid"); + return LW_STATUS_INVALID_ARGUMENT; + } + profiling = rec_profile_enabled(); +#if defined(__EMSCRIPTEN__) + /* CLS shares this executor but has no CTC tail: never pool it into REC. */ + detailed = rec_op_profile_enabled() && instance->program->ctc.enabled != 0u; + if (detailed) { + invocation = ++next_invocation; + fprintf(stderr, "WASM_REC_OP_BEGIN width=%u invocation=%llu ops=%u\n", + instance->program->target_width, (unsigned long long)invocation, + instance->program->op_count); + profiling = 1; + } +#endif + if (profiling) { + memset(&instance->profile, 0, sizeof(instance->profile)); + } + for (i = 0u; i < instance->program->op_count; ++i) { + int paired = 0; +#if defined(LW_EXPERIMENTAL_REC_FFN_TILING) + paired = (instance->program->ops[i].flags & LW_X64_REC_OP_FFN_BEGIN) != 0u; +#endif +#if defined(__EMSCRIPTEN__) + /* Detailed one-op traces retain the unfused diagnostic execution. */ + if (detailed) paired = 0; +#endif + if (profiling) { + uint64_t started = rec_profile_now_ns(); + uint64_t* slot; + status = paired ? lw_x64_rec_instance_run_ffn_pair(instance, i, error) + : execute_op(instance, &instance->program->ops[i], error); + slot = rec_profile_slot(&instance->profile, instance->program->ops[i].kind); + { + uint64_t finished = rec_profile_now_ns(); + uint64_t elapsed = finished >= started ? finished - started : 0u; + if (slot != NULL) *slot += elapsed; + instance->profile.total_ns += elapsed; + if (paired) { + instance->profile.ffn_ns += elapsed; + ++instance->profile.ffn_calls; + } else if (instance->program->ops[i].kind == LW_X64_REC_OP_POINTWISE) { + record_rec_pointwise_epilogue(&instance->profile, + &instance->program->ops[i], elapsed); + } +#if defined(__EMSCRIPTEN__) + if (detailed) rec_op_profile_record(instance->program, i, invocation, elapsed, status); +#endif + } + } else { + status = paired ? lw_x64_rec_instance_run_ffn_pair(instance, i, error) + : execute_op(instance, &instance->program->ops[i], error); + } + if (status != LW_STATUS_OK) { + if (error != NULL && error->message[0] == '\0') { + char message[96]; + (void)snprintf(message, sizeof(message), "REC physical op %u kind %u failed", (unsigned)i, (unsigned)instance->program->ops[i].kind); + lw_set_error(error, status, message); + } + return status; + } + if (paired) ++i; + } +#if defined(__EMSCRIPTEN__) + if (detailed) fprintf(stderr, "WASM_REC_OP_END width=%u invocation=%llu\n", + instance->program->target_width, (unsigned long long)invocation); +#endif + return LW_STATUS_OK; +} + +lw_status lw_x64_rec_instance_run_op(lw_x64_rec_instance* instance, uint32_t op_index, lw_error* error) { + if (instance == NULL || instance->program == NULL || op_index >= instance->program->op_count) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "REC physical op index is invalid"); + return LW_STATUS_INVALID_ARGUMENT; + } + return execute_op(instance, &instance->program->ops[op_index], error); +} +lw_status lw_x64_rec_instance_run_backbone(lw_x64_rec_instance* instance, lw_error* error) { + lw_status status = run_backbone_ops(instance, error); + if (status == LW_STATUS_OK) lw_set_error(error, LW_STATUS_OK, ""); + return status; +} + +lw_status lw_x64_rec_instance_run(lw_x64_rec_instance* instance, lw_error* error) { + lw_status status; + float* activation; + int profiling = rec_profile_enabled(); + status = run_backbone_ops(instance, error); + if (status != LW_STATUS_OK) return status; + if (instance->program->ctc.enabled) { + uint64_t ctc_started = profiling ? rec_profile_now_ns() : 0u; + activation = offset_ptr(instance, instance->program->values[instance->program->ctc.activation_value].offset); + status = lw_x64_rec_ctc_execute(instance->program, instance, activation, error); + if (status != LW_STATUS_OK) return status; + if (profiling) { + instance->profile.ctc_ns = rec_profile_now_ns() - ctc_started; + } + } + if (profiling) { + const lw_x64_rec_profile* p = &instance->profile; + fprintf(stderr, + "X64REC width=%u total=%.3f pw=%.3f dense=%.3f dw=%.3f bin=%.3f unary=%.3f " + "reduce=%.3f pool=%.3f transpose=%.3f matmul=%.3f ctc=%.3f\n", + (unsigned)instance->program->target_width, p->total_ns / 1e6, + p->pointwise_ns / 1e6, p->dense_ns / 1e6, p->depthwise_ns / 1e6, + p->binary_ns / 1e6, p->unary_ns / 1e6, p->reduce_ns / 1e6, + p->pool_ns / 1e6, p->transpose_ns / 1e6, p->matmul_ns / 1e6, + p->ctc_ns / 1e6); + if (p->ffn_calls != 0u) + fprintf(stderr, "REC_FFN width=%u pairs=%u elapsed=%.3f tile=%u\n", + instance->program->target_width, p->ffn_calls, p->ffn_ns / 1e6, + LW_REC_FFN_TILE_PIXELS); +#if defined(__EMSCRIPTEN__) + { + static const char* const names[REC_PW_EPI_COUNT] = { + "plain", "bias", "hardswish", "residual", "resgelu", "postgelu", "other" + }; + uint32_t kind; + for (kind = 0u; kind < REC_PW_EPI_COUNT; ++kind) { + if (p->pw_epi_calls[kind] == 0u) continue; + fprintf(stderr, "WASM_PW_EPI width=%u kind=%s calls=%llu macs=%llu elapsed=%.3f\n", + (unsigned)instance->program->target_width, names[kind], + (unsigned long long)p->pw_epi_calls[kind], + (unsigned long long)p->pw_epi_macs[kind], + p->pw_epi_ns[kind] / 1e6); + } + } +#endif + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_internal.h new file mode 100644 index 00000000..d6007d34 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_backend_internal.h @@ -0,0 +1,468 @@ +#ifndef LW_X64_REC_BACKEND_INTERNAL_H +#define LW_X64_REC_BACKEND_INTERNAL_H + +#include "cpu_features.h" +#include "atomic_internal.h" +#include "model_internal.h" +#include "session_internal.h" +#include "executor_internal.h" +#include "../kernels/nhwc_internal.h" +#include "../kernels/packed_conv_internal.h" +#include "../kernels/packed_conv3x3_internal.h" +#include "../kernels/scalar_kernels.h" + +#include +#include + +/* Keep the individual physical ops for diagnostic/reference execution. */ +#define LW_X64_REC_OP_FFN_BEGIN 1u + +typedef enum lw_x64_rec_compile_result { + LW_X64_REC_COMPILE_OK = 0, + LW_X64_REC_COMPILE_UNSUPPORTED = 1, + LW_X64_REC_COMPILE_INVALID_GRAPH = 2, + LW_X64_REC_COMPILE_OUT_OF_MEMORY = 3 +} lw_x64_rec_compile_result; + +typedef enum lw_x64_rec_storage_layout { + LW_X64_REC_LAYOUT_NHWC = 0, + LW_X64_REC_LAYOUT_TC = 1, + LW_X64_REC_LAYOUT_VECTOR = 2, + LW_X64_REC_LAYOUT_SCALAR = 3, + LW_X64_REC_LAYOUT_NCHW = 4 +} lw_x64_rec_storage_layout; + +typedef enum lw_x64_rec_backend_layout { + LW_X64_REC_BACKEND_NHWC = 0, + LW_X64_REC_BACKEND_NCHW = 1 +} lw_x64_rec_backend_layout; + +typedef enum lw_x64_rec_compile_strategy { + LW_X64_REC_COMPILE_NHWC = 0, + LW_X64_REC_COMPILE_NCHW = 1 +} lw_x64_rec_compile_strategy; + +typedef enum lw_x64_rec_op_kind { + LW_X64_REC_OP_DENSE = 1, + LW_X64_REC_OP_POINTWISE = 2, + LW_X64_REC_OP_DEPTHWISE = 3, + LW_X64_REC_OP_POINTWISE_NCHW = 19, + LW_X64_REC_OP_STEM_NCHW = 20, + LW_X64_REC_OP_DEPTHWISE_NCHW = 21, + LW_X64_REC_OP_AFFINE = 4, + LW_X64_REC_OP_ADD = 5, + LW_X64_REC_OP_MUL = 6, + LW_X64_REC_OP_DIV = 7, + LW_X64_REC_OP_RELU = 8, + LW_X64_REC_OP_ERF = 9, + LW_X64_REC_OP_GELU = 10, + LW_X64_REC_OP_HARD_SIGMOID = 11, + LW_X64_REC_OP_REDUCE_MEAN = 12, + LW_X64_REC_OP_AVG_POOL = 13, + LW_X64_REC_OP_MAX_POOL = 14, + LW_X64_REC_OP_TRANSPOSE = 15, + LW_X64_REC_OP_MATMUL = 16, + LW_X64_REC_OP_CONCAT = 17, + LW_X64_REC_OP_RESIZE = 18, + LW_X64_REC_OP_SOFTMAX = 22, + LW_X64_REC_OP_SUB = 23, + LW_X64_REC_OP_POW = 24, + LW_X64_REC_OP_SIGMOID = 25, + LW_X64_REC_OP_SQRT = 26, + LW_X64_REC_OP_SLICE = 27, + LW_X64_REC_OP_GENERIC_UNSUPPORTED = 0xffff +} lw_x64_rec_op_kind; + +typedef enum lw_x64_rec_conv_kind { + LW_X64_REC_CONV_POINTWISE = 1, + LW_X64_REC_CONV_DEPTHWISE = 2, + LW_X64_REC_CONV_DENSE = 3 +} lw_x64_rec_conv_kind; + +typedef enum lw_x64_rec_pointwise_kernel { + LW_X64_REC_PW_6X16 = 0, + LW_X64_REC_PW_4X16 = 1, + LW_X64_REC_PW_3X32 = 2, + LW_X64_REC_PW_2X32 = 3 +} lw_x64_rec_pointwise_kernel; + +typedef enum lw_x64_rec_nchw_pointwise_kernel { + LW_X64_REC_NCHW_PW_FMA4 = 0, + LW_X64_REC_NCHW_PW_FMA8 = 1 +} lw_x64_rec_nchw_pointwise_kernel; + +typedef enum lw_x64_rec_broadcast_kind { + LW_X64_REC_BROADCAST_SAME = 0, + LW_X64_REC_BROADCAST_RIGHT_SCALAR = 1, + LW_X64_REC_BROADCAST_LEFT_SCALAR = 2, + LW_X64_REC_BROADCAST_RIGHT_CHANNEL = 3, + LW_X64_REC_BROADCAST_LEFT_CHANNEL = 4, + LW_X64_REC_BROADCAST_GENERAL = 5, + /* Right operand holds one scalar per output pixel (element_count / + * channels floats), broadcast over the channel axis. Bit-identical + * to the canonical scalar broadcast for every operation. */ + LW_X64_REC_BROADCAST_RIGHT_PIXEL_SCALAR = 6 +} lw_x64_rec_broadcast_kind; + +typedef struct lw_x64_rec_value { + uint64_t offset; + uint64_t bytes; + const float* constant_data; + int32_t producer; + int32_t last_use; + int32_t dimensions[4]; + uint32_t rank; + uint8_t layout; + uint8_t alias; + uint16_t reserved; + uint32_t alias_of; +} lw_x64_rec_value; + +typedef struct lw_x64_rec_conv_op { + uint64_t input_offset; + uint64_t output_offset; + const float* packed_weights; + const float* original_weights; + const float* bias; + uint8_t scalar_fallback; + uint8_t pointwise_kernel; + uint8_t nchw_pointwise_kernel; + uint8_t conv_reserved; + uint32_t input_channels; + uint32_t output_channels; + uint32_t input_height; + uint32_t input_width; + uint32_t output_height; + uint32_t output_width; + uint32_t kernel_h; + uint32_t kernel_w; + uint32_t weight_h; + uint32_t weight_w; + uint32_t stride_h; + uint32_t stride_w; + uint32_t pad_top; + uint32_t pad_left; + uint32_t pad_bottom; + uint32_t pad_right; + uint32_t groups; + uint32_t dense_kc; + uint64_t scratch_bytes; + uint16_t kernel_kind; + uint16_t activation; + /* Fused epilogue state filled by the compiler's conv -> add -> gelu + * matcher. post_bias is a folded channel bias applied after accumulation; + * residual_offset/has_residual describe a fused elementwise residual add + * that reads from the arena. All zero when nothing was folded in. */ + const float* post_bias; + uint64_t residual_offset; + uint8_t has_residual; + uint8_t fusion_reserved[7]; +} lw_x64_rec_conv_op; + +typedef struct lw_x64_rec_binary_op { + uint64_t left_offset; + uint64_t right_offset; + uint64_t output_offset; + const float* left_constant; + const float* right_constant; + uint64_t element_count; + uint32_t pixels; + uint32_t channels; + uint16_t operation; + uint8_t broadcast_kind; + uint8_t mixed_nhwc; /* 0=uniform layout, 1=right operand physical NHWC, 2=left */ + /* 1 = channel broadcast over an NCHW-physical rank-4 tensor (channel axis + * is dims[1]); 0 = channel-last or non-channel broadcast. The NCHW + * executor branch keys on this flag so canonical rank-3 [.., .., C] + * tensors keep the channel-last path. */ + uint8_t channel_major_nchw; + int32_t dimensions[4]; +} lw_x64_rec_binary_op; + +typedef struct lw_x64_rec_unary_op { + uint64_t input_offset; + uint64_t output_offset; + uint64_t element_count; + float alpha; + float beta; + int32_t dimensions[4]; + uint32_t rank; +} lw_x64_rec_unary_op; + +typedef struct lw_x64_rec_affine_op { + uint64_t input_offset; + uint64_t output_offset; + const float* mul; + const float* add; + const float* scale; + const float* bias; + const float* mean; + const float* variance; + float epsilon; + uint8_t channel_major; + uint32_t pixels; + uint32_t channels; +} lw_x64_rec_affine_op; + +typedef struct lw_x64_rec_reduce_op { + uint64_t input_offset; + uint64_t output_offset; + uint32_t batch; + uint32_t height; + uint32_t width; + uint32_t channels; + /* general != 0: not the NCHW/NHWC global-pool pattern; run the + * canonical scalar reduce with the original axes for bit-identity. */ + uint32_t general; + uint32_t input_rank; + uint32_t output_rank; + uint32_t axes_count; + uint32_t keep_dimensions; + uint32_t no_op_with_empty_axes; + int32_t axes[LW_MAX_DIMS]; + int32_t input_dimensions[LW_MAX_DIMS]; + int32_t output_dimensions[LW_MAX_DIMS]; +} lw_x64_rec_reduce_op; + +typedef struct lw_x64_rec_pool_op { + uint64_t input_offset; + uint64_t output_offset; + int32_t input_dimensions[4]; + int32_t output_dimensions[4]; + int32_t kernel[2]; + int32_t strides[2]; + int32_t pads[4]; + uint8_t count_include_pad; + uint8_t is_max; +} lw_x64_rec_pool_op; + +typedef struct lw_x64_rec_transpose_op { + uint64_t input_offset; + uint64_t output_offset; + uint32_t rank; + int32_t input_dimensions[LW_MAX_DIMS]; + int32_t output_dimensions[LW_MAX_DIMS]; + int32_t permutation[LW_MAX_DIMS]; +} lw_x64_rec_transpose_op; + +typedef struct lw_x64_rec_matmul_op { + uint64_t input_offset; + uint64_t output_offset; + uint64_t weights_offset; + const float* weights; + float* packed_weights; + uint32_t batch; + uint32_t rows; + uint32_t inner; + uint32_t columns; + uint32_t general; + uint32_t input_rank; + uint32_t weights_rank; + uint32_t output_rank; + int32_t input_dimensions[LW_MAX_DIMS]; + int32_t weights_dimensions[LW_MAX_DIMS]; + int32_t output_dimensions[LW_MAX_DIMS]; +} lw_x64_rec_matmul_op; + +typedef struct lw_x64_rec_softmax_op { + uint64_t input_offset; + uint64_t output_offset; + int32_t axis; + uint32_t rank; + int32_t dimensions[4]; +} lw_x64_rec_softmax_op; + +/* Channel-axis (NCHW axis 1) concat over rank-4 feature maps stored NHWC: + * per-pixel concatenated channel copies, pure data movement. */ +typedef struct lw_x64_rec_concat_op { + uint64_t input_offsets[LWM_V0_MAX_NODE_INPUTS]; + uint64_t output_offset; + uint32_t input_channels[LWM_V0_MAX_NODE_INPUTS]; + uint32_t input_count; + uint32_t output_channels; + uint32_t pixels; + int32_t axis; +} lw_x64_rec_concat_op; + +/* Generic slice over a canonical-layout (non-NHWC) tensor; executed by + * the shared scalar slice kernel so results match the interpreter. */ +typedef struct lw_x64_rec_slice_op { + uint64_t input_offset; + uint64_t output_offset; + uint32_t rank; + uint32_t output_rank; + uint32_t slice_count; + int32_t input_dimensions[LW_MAX_DIMS]; + int32_t output_dimensions[LW_MAX_DIMS]; + int32_t starts[LW_MAX_DIMS]; + int32_t ends[LW_MAX_DIMS]; + int32_t axes[LW_MAX_DIMS]; + int32_t steps[LW_MAX_DIMS]; +} lw_x64_rec_slice_op; + +typedef struct lw_x64_rec_op { + uint16_t kind; + uint16_t reserved; + uint32_t semantic_begin; + uint16_t semantic_count; + uint16_t flags; + union { + lw_x64_rec_conv_op conv; + lw_x64_rec_binary_op binary; + lw_x64_rec_unary_op unary; + lw_x64_rec_affine_op affine; + lw_x64_rec_reduce_op reduce; + lw_x64_rec_pool_op pool; + lw_x64_rec_transpose_op transpose; + lw_x64_rec_matmul_op matmul; + lw_x64_rec_softmax_op softmax; + lw_x64_rec_concat_op concat; + lw_x64_rec_slice_op slice; + } data; +} lw_x64_rec_op; + +typedef struct lw_x64_rec_constant { + void* data; + uint64_t bytes; +} lw_x64_rec_constant; + +typedef struct lw_x64_rec_ctc_tail { + uint8_t enabled; + uint8_t packed_weights_borrowed; + uint8_t reserved[2]; + uint32_t activation_value; + uint32_t classes; + uint32_t rows; + uint32_t inner; + uint32_t weight_tensor; + uint32_t bias_tensor; + float* packed_weights; + const float* bias; +} lw_x64_rec_ctc_tail; + +typedef struct lw_x64_rec_program { + const lw_model* model; + lw_cpu_capabilities cpu; + uint64_t model_signature; + uint32_t target_width; + uint32_t time_steps; + uint32_t class_count; + uint32_t input_value; + uint32_t output_value; + lw_x64_rec_op* ops; + uint32_t op_count; + lw_x64_rec_value* values; + uint32_t value_count; + uint32_t semantic_consumed; + uint32_t semantic_elided; + uint32_t semantic_fused; + uint32_t unsupported_nodes; + uint32_t generic_ops; + uint32_t layout_conversions; + uint8_t direct_nhwc; + uint8_t ctc_fused; + uint16_t reserved; + uint64_t arena_bytes; + uint64_t scratch_bytes; + uint8_t backend_layout; + uint32_t packed_constant_count; + lw_x64_rec_constant* constants; + const struct lw_x64_rec_program* packed_source; + uint64_t owned_constant_bytes; + uint64_t borrowed_constant_bytes; + uint32_t borrowed_constant_count; + lw_x64_rec_ctc_tail ctc; + /* Number of additional owners beyond the original compiler caller. */ + lw_atomic_u32 shared_refs; +} lw_x64_rec_program; + +typedef struct lw_x64_rec_profile { + uint64_t total_ns; + uint64_t pointwise_ns; + uint64_t dense_ns; + uint64_t depthwise_ns; + uint64_t binary_ns; + uint64_t unary_ns; + uint64_t reduce_ns; + uint64_t pool_ns; + uint64_t transpose_ns; + uint64_t matmul_ns; + uint64_t ctc_ns; + uint64_t ffn_ns; + uint32_t ffn_calls; + /* Pointwise epilogue diagnostics; populated only when REC profiling is on. */ + uint64_t pw_epi_ns[7]; + uint64_t pw_epi_calls[7]; + uint64_t pw_epi_macs[7]; +} lw_x64_rec_profile; + +typedef struct lw_x64_rec_instance { + const lw_x64_rec_program* program; + uint8_t* arena; + uint8_t* scratch; + float* ctc_scores; + uint32_t* best_indices; + float* best_probabilities; + lw_x64_rec_profile profile; + /* Borrowed intra-op pool for the CTC head matmul (NULL = serial). + * Owned by the recognizer; every slot instance of one recognizer + * shares it, and slots never run concurrently. */ + lw_thread_pool* thread_pool; + uint32_t intra_op_workers; + /* Lazily allocated per-worker scratch for sharded dense convs + * (intra_op_workers * per-op scratch bytes, grown on demand). + * Always owned by the instance, independent of owns_workspace. */ + uint8_t* dense_shard_scratch; + uint64_t dense_shard_scratch_bytes; + /* 0 when arena/scratch/CTC buffers are borrowed from a shared + * cross-width workspace owned by the recognizer. */ + uint8_t owns_workspace; +} lw_x64_rec_instance; + +uint64_t lw_x64_rec_arena_align(uint64_t value, uint64_t alignment); +lw_status lw_x64_rec_arena_alloc(uint64_t* cursor, uint64_t bytes, uint64_t alignment, + uint64_t* out_offset, lw_error* error); +/* Generalized entry: explicit input height (REC callers pass 48; the CLS + * backbone reuses this lowering with its own fixed input shape). */ +lw_x64_rec_compile_result lw_x64_rec_backend_compile_input( + const lw_model* model, uint32_t input_height, uint32_t target_width, + lw_x64_rec_compile_strategy strategy, uint32_t allow_ctc_tail, + lw_x64_rec_program** out_program, lw_error* error); +lw_x64_rec_compile_result lw_x64_rec_backend_compile_ex( + const lw_model* model, uint32_t target_width, + lw_x64_rec_compile_strategy strategy, + lw_x64_rec_program** out_program, lw_error* error); +lw_x64_rec_compile_result lw_x64_rec_backend_compile( + const lw_model* model, uint32_t target_width, lw_x64_rec_program** out_program, + lw_error* error); +lw_x64_rec_compile_result lw_x64_rec_backend_compile_shared( + const lw_model* model, uint32_t target_width, const lw_x64_rec_program* packed_source, + lw_x64_rec_program** out_program, lw_error* error); +void lw_x64_rec_program_free(lw_x64_rec_program* program); +void lw_x64_rec_program_retain(lw_x64_rec_program* program); +lw_status lw_x64_rec_instance_create(const lw_x64_rec_program* program, + lw_x64_rec_instance** out, lw_error* error); +/* Borrowed-workspace variant: the instance uses the caller's buffers + * (sized at least to the program's requirements) and never frees + * them. ctc_rows is the capacity of the three CTC arrays. */ +lw_status lw_x64_rec_instance_create_borrowed( + const lw_x64_rec_program* program, + uint8_t* arena, uint64_t arena_bytes, + uint8_t* scratch, uint64_t scratch_bytes, + float* ctc_scores, uint32_t* best_indices, float* best_probabilities, + uint64_t ctc_rows, lw_x64_rec_instance** out, lw_error* error); +void lw_x64_rec_instance_free(lw_x64_rec_instance* instance); +/* Attach a borrowed intra-op pool for the CTC head projection + * (workers <= 1 or NULL pool detaches; serial fallback). */ +void lw_x64_rec_instance_set_thread_pool(lw_x64_rec_instance* instance, + lw_thread_pool* pool, uint32_t workers); +float* lw_x64_rec_instance_input(lw_x64_rec_instance* instance, uint64_t* element_count); +lw_status lw_x64_rec_instance_run(lw_x64_rec_instance* instance, lw_error* error); +lw_status lw_x64_rec_instance_run_backbone(lw_x64_rec_instance* instance, lw_error* error); +/* Test-only internal hook: execute one compiled physical op. */ +lw_status lw_x64_rec_instance_run_op(lw_x64_rec_instance* instance, uint32_t op_index, lw_error* error); +/* Private contract hook: execute an eligible adjacent pair in cache-local tiles. */ +lw_status lw_x64_rec_instance_run_ffn_pair(lw_x64_rec_instance* instance, + uint32_t op_index, lw_error* error); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_fast_executor.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_fast_executor.c new file mode 100644 index 00000000..02cc9eb3 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_fast_executor.c @@ -0,0 +1,798 @@ +#include "x64_rec_fast_internal.h" + +#include "executor_internal.h" +#include "operator_internal.h" +#include "lwm_read.h" +#include "../simd/simd_kernels.h" + +#include +#include +#include +#include +#include + +static const float* fast_elementwise_constant_f32(const lw_session* session, uint32_t tensor_index) { + const lw_runtime_tensor* tensor; + const uint8_t* disk; + uint64_t data_offset; + if (session == NULL || session->model == NULL || tensor_index >= session->model->info.tensor_count) return NULL; + tensor = &session->tensors[tensor_index]; + if (tensor->dtype != LW_DTYPE_F32 || (tensor->flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u) return NULL; + disk = session->model->bytes + (size_t)session->model->tensor_offset + + (size_t)tensor_index * LWM_V0_TENSOR_SIZE; + data_offset = lwm_read_u64(disk + 48u); + if (data_offset > session->model->byte_count || tensor->byte_size > session->model->byte_count - data_offset) return NULL; + return (const float*)(const void*)(session->model->bytes + (size_t)data_offset); +} +static uint64_t fast_element_count(const lw_runtime_tensor* tensor) { + return tensor == NULL || tensor->dtype != LW_DTYPE_F32 ? 0u : tensor->byte_size / sizeof(float); +} + +static float* fast_nhwc_pointer(const lw_x64_rec_fast_plan* plan, uint32_t tensor_index) { + const lw_x64_fast_tensor_state* state; + if (plan == NULL || tensor_index >= plan->tensor_count) { + return NULL; + } + state = &plan->tensors[tensor_index]; + if (state->neutral) { + return (float*)lw_executor_tensor_output_data(plan->session, tensor_index); + } + if (state->nhwc_offset == LW_X64_FAST_OFFSET_NONE || plan->nhwc_workspace == NULL) { + return NULL; + } + return (float*)(void*)(plan->nhwc_workspace + (size_t)state->nhwc_offset); +} + +static int fast_layout_tracked_tensor(const lw_runtime_tensor* tensor) { + return tensor != NULL && tensor->dtype == LW_DTYPE_F32 && tensor->rank == 4u && + tensor->dimensions[0] > 0 && tensor->dimensions[1] > 0 && + tensor->dimensions[2] > 0 && tensor->dimensions[3] > 0; +} + +static void fast_publish_nchw(lw_x64_rec_fast_plan* plan, uint32_t tensor_index) { + lw_x64_fast_tensor_state* state; + if (plan == NULL || tensor_index >= plan->tensor_count) return; + state = &plan->tensors[tensor_index]; + state->available_layouts = state->neutral + ? (uint8_t)(LW_X64_FAST_HAVE_NCHW | LW_X64_FAST_HAVE_NHWC) + : LW_X64_FAST_HAVE_NCHW; +} + +static void fast_publish_nhwc(lw_x64_rec_fast_plan* plan, uint32_t tensor_index) { + lw_x64_fast_tensor_state* state; + if (plan == NULL || tensor_index >= plan->tensor_count) return; + state = &plan->tensors[tensor_index]; + state->available_layouts = state->neutral + ? (uint8_t)(LW_X64_FAST_HAVE_NCHW | LW_X64_FAST_HAVE_NHWC) + : LW_X64_FAST_HAVE_NHWC; +} + +static void fast_begin_run(lw_x64_rec_fast_plan* plan, + lw_x64_fast_input_layout input_layout) { + uint32_t index; + if (plan == NULL) return; + plan->conversion_count = 0u; + plan->conversion_bytes = 0u; + plan->depthwise_x2_invocations = 0u; + plan->depthwise_x1_invocations = 0u; + for (index = 0u; index < plan->tensor_count; ++index) { + plan->tensors[index].available_layouts = 0u; + } + if (input_layout == LW_X64_FAST_INPUT_NHWC) { + fast_publish_nhwc(plan, plan->graph_input_index); + } else { + fast_publish_nchw(plan, plan->graph_input_index); + } +} + +float* lw_x64_rec_fast_graph_input_nhwc(lw_x64_rec_fast_plan* plan, + uint64_t* element_count) { + const lw_runtime_tensor* tensor; + if (element_count != NULL) *element_count = 0u; + if (plan == NULL || plan->graph_input_index >= plan->tensor_count || + plan->layout.graph_input_direct_nhwc == 0u) return NULL; + tensor = &plan->session->tensors[plan->graph_input_index]; + if (tensor->dtype != LW_DTYPE_F32 || tensor->rank != 4u) return NULL; + if (element_count != NULL) *element_count = tensor->byte_size / sizeof(float); + return fast_nhwc_pointer(plan, plan->graph_input_index); +} + +static void fast_add_conversion(lw_x64_rec_fast_plan* plan, uint64_t bytes) { + if (plan->conversion_count != UINT64_MAX) ++plan->conversion_count; + if (plan->conversion_bytes > UINT64_MAX - bytes) plan->conversion_bytes = UINT64_MAX; + else plan->conversion_bytes += bytes; +} + +static lw_status fast_ensure_nhwc(lw_x64_rec_fast_plan* plan, uint32_t tensor_index, + uint32_t graph_input_index, const float* graph_input, + lw_error* error) { + const lw_runtime_tensor* tensor; + const float* source; + float* destination; + if (plan == NULL || tensor_index >= plan->tensor_count) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid fast tensor index"); + return LW_STATUS_INVALID_ARGUMENT; + } + if ((plan->tensors[tensor_index].available_layouts & LW_X64_FAST_HAVE_NHWC) != 0u) { + return LW_STATUS_OK; + } + tensor = &plan->session->tensors[tensor_index]; + if (tensor->rank != 4u || tensor->dtype != LW_DTYPE_F32 || + (plan->tensors[tensor_index].available_layouts & LW_X64_FAST_HAVE_NCHW) == 0u) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "tensor cannot be converted to NHWC"); + return LW_STATUS_UNSUPPORTED; + } + source = lw_executor_tensor_input_data(plan->session, tensor_index, graph_input_index, + graph_input); + destination = fast_nhwc_pointer(plan, tensor_index); + if (source == NULL || destination == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "NCHW to NHWC conversion failed"); + return LW_STATUS_UNSUPPORTED; + } + lw_x64_fast_nchw_to_nhwc(source, destination, (uint32_t)tensor->dimensions[0], + (uint32_t)tensor->dimensions[1], (uint32_t)tensor->dimensions[2], + (uint32_t)tensor->dimensions[3]); + plan->tensors[tensor_index].available_layouts = (uint8_t)(LW_X64_FAST_HAVE_NCHW | LW_X64_FAST_HAVE_NHWC); + fast_add_conversion(plan, tensor->byte_size); + return LW_STATUS_OK; +} + +static lw_status fast_ensure_nchw(lw_x64_rec_fast_plan* plan, uint32_t tensor_index, + lw_error* error) { + const lw_runtime_tensor* tensor; + const float* source; + float* destination; + if (plan == NULL || tensor_index >= plan->tensor_count) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid fast tensor index"); + return LW_STATUS_INVALID_ARGUMENT; + } + if ((plan->tensors[tensor_index].available_layouts & LW_X64_FAST_HAVE_NCHW) != 0u) { + return LW_STATUS_OK; + } + tensor = &plan->session->tensors[tensor_index]; + if (tensor->rank != 4u || tensor->dtype != LW_DTYPE_F32) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "tensor cannot be converted to NCHW"); + return LW_STATUS_UNSUPPORTED; + } + source = fast_nhwc_pointer(plan, tensor_index); + destination = lw_executor_tensor_output_data(plan->session, tensor_index); + if (source == NULL || destination == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "NHWC to NCHW conversion failed"); + return LW_STATUS_UNSUPPORTED; + } + lw_x64_fast_nhwc_to_nchw(source, destination, (uint32_t)tensor->dimensions[0], + (uint32_t)tensor->dimensions[1], (uint32_t)tensor->dimensions[2], + (uint32_t)tensor->dimensions[3]); + plan->tensors[tensor_index].available_layouts = (uint8_t)(LW_X64_FAST_HAVE_NCHW | LW_X64_FAST_HAVE_NHWC); + fast_add_conversion(plan, tensor->byte_size); + return LW_STATUS_OK; +} + +static lw_status fast_execute_generic_node(lw_x64_rec_fast_plan* plan, uint32_t node_index, + uint32_t graph_input_index, const float* graph_input, + uint32_t* consumed_nodes, lw_error* error) { + const uint8_t* node; + uint16_t input_count; + uint32_t input_slot; + uint32_t output_index; + uint32_t consumed = 1u; + lw_status status; + if (plan == NULL || node_index >= plan->node_count) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid fast node index"); + return LW_STATUS_INVALID_ARGUMENT; + } + node = plan->session->model->bytes + (size_t)plan->session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + input_count = lwm_read_u16(node + 2u); + if (input_count > LWM_V0_MAX_NODE_INPUTS) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "node input count exceeds limit"); + return LW_STATUS_INVALID_SHAPE; + } + for (input_slot = 0u; input_slot < input_count; ++input_slot) { + uint32_t input_index = lwm_read_u32(node + 8u + (size_t)input_slot * sizeof(uint32_t)); + if (input_index >= plan->tensor_count) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "node input index out of bounds"); + return LW_STATUS_OUT_OF_BOUNDS; + } + if ((plan->session->tensors[input_index].flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u && + fast_layout_tracked_tensor(&plan->session->tensors[input_index])) { + status = fast_ensure_nchw(plan, input_index, error); + if (status != LW_STATUS_OK) return status; + } + } + status = lw_executor_execute_best_node_f32(plan->session, node_index, graph_input_index, + graph_input, NULL, &consumed); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "generic fast executor node failed"); + return status; + } + if (consumed == 0u || consumed > plan->node_count - node_index) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "legacy node consumption is invalid"); + return LW_STATUS_OUT_OF_BOUNDS; + } + output_index = lwm_read_u32(plan->session->model->bytes + + (size_t)plan->session->model->node_offset + + (size_t)(node_index + consumed - 1u) * LWM_V0_NODE_SIZE + 40u); + if (output_index >= plan->tensor_count) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "node output index out of bounds"); + return LW_STATUS_OUT_OF_BOUNDS; + } + if (fast_layout_tracked_tensor(&plan->session->tensors[output_index])) { + fast_publish_nchw(plan, output_index); + } + if (consumed_nodes != NULL) *consumed_nodes = consumed; + return LW_STATUS_OK; +} +static lw_status fast_execute_pointwise(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* node, + uint32_t graph_input_index, + const float* graph_input, + lw_error* error) { + const lw_x64_fast_conv* conv = &node->data.conv; + float* input; + float* output; + lw_nhwc_epilogue epilogue; + lw_status status; + status = fast_ensure_nhwc(plan, conv->input_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + input = fast_nhwc_pointer(plan, conv->input_index); + output = fast_nhwc_pointer(plan, conv->output_index); + if (input == NULL || output == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "pointwise fast buffers are unavailable"); + return LW_STATUS_UNSUPPORTED; + } + memset(&epilogue, 0, sizeof(epilogue)); + epilogue.bias = conv->bias; + epilogue.activation = conv->activation; + if (conv->residual_index != UINT32_MAX) { + status = fast_ensure_nhwc(plan, conv->residual_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + epilogue.residual = fast_nhwc_pointer(plan, conv->residual_index); + if (epilogue.residual == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "pointwise residual buffer is unavailable"); + return LW_STATUS_UNSUPPORTED; + } + } + lw_avx2_fma_nhwc_pointwise_f32(input, conv->packed_weights, &epilogue, output, + conv->input_height * conv->input_width, + conv->input_channels, conv->output_channels); + fast_publish_nhwc(plan, conv->output_index); + return LW_STATUS_OK; +} + +static lw_status fast_execute_dense(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* node, + uint32_t graph_input_index, + const float* graph_input, + lw_error* error) { + const lw_x64_fast_conv* conv = &node->data.conv; + lw_nhwc_dense_desc desc; + lw_nhwc_epilogue epilogue; + float* input; + float* output; + lw_status status; + status = fast_ensure_nhwc(plan, conv->input_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + input = fast_nhwc_pointer(plan, conv->input_index); + output = fast_nhwc_pointer(plan, conv->output_index); + if (input == NULL || output == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "dense fast buffers are unavailable"); + return LW_STATUS_UNSUPPORTED; + } + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; + desc.input_channels = conv->input_channels; + desc.input_height = conv->input_height; + desc.input_width = conv->input_width; + desc.output_channels = conv->output_channels; + desc.output_height = conv->output_height; + desc.output_width = conv->output_width; + desc.kernel_h = conv->kernel_h; + desc.kernel_w = conv->kernel_w; + desc.stride_h = conv->stride_h; + desc.stride_w = conv->stride_w; + desc.pad_top = conv->pad_top; + desc.pad_left = conv->pad_left; + desc.pad_bottom = conv->pad_bottom; + desc.pad_right = conv->pad_right; + desc.dense_kc = conv->dense_kc; + memset(&epilogue, 0, sizeof(epilogue)); + epilogue.bias = conv->bias; + epilogue.activation = conv->activation; + if (conv->residual_index != UINT32_MAX) { + status = fast_ensure_nhwc(plan, conv->residual_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + epilogue.residual = fast_nhwc_pointer(plan, conv->residual_index); + if (epilogue.residual == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "dense residual buffer is unavailable"); + return LW_STATUS_UNSUPPORTED; + } + } + status = lw_avx2_fma_nhwc_dense_prepared_f32(input, conv->packed_weights, + conv->dense_input_offsets_k, conv->dense_patch_offsets_k, &epilogue, output, + &desc, plan->scratch, plan->scratch_bytes); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "dense fast kernel failed"); + return status; + } + fast_publish_nhwc(plan, conv->output_index); + return LW_STATUS_OK; +} + +static lw_status fast_elementwise_scalar_value(lw_x64_rec_fast_plan* plan, uint32_t tensor_index, + uint32_t graph_input_index, const float* graph_input, + float* value, lw_error* error) { + const float* data; + lw_status status; + if (value == NULL || tensor_index >= plan->tensor_count) return LW_STATUS_INVALID_ARGUMENT; + data = fast_elementwise_constant_f32(plan->session, tensor_index); + if (data == NULL) { + status = fast_ensure_nhwc(plan, tensor_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + data = fast_nhwc_pointer(plan, tensor_index); + } + if (data == NULL) return LW_STATUS_UNSUPPORTED; + *value = data[0]; + return LW_STATUS_OK; +} + +static lw_status fast_execute_elementwise(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* node, + uint32_t graph_input_index, + const float* graph_input, + lw_error* error) { + const lw_x64_fast_elementwise* op = &node->data.elementwise; + const lw_runtime_tensor* output_tensor = &plan->session->tensors[op->output_index]; + float* input = NULL; + float* output; + uint64_t elements = output_tensor->byte_size / sizeof(float); + lw_status status; + if (node->kind == LW_X64_FAST_NODE_CONTIGUOUS_UNARY || + op->broadcast_kind == LW_X64_FAST_BROADCAST_NONE || + op->broadcast_kind == LW_X64_FAST_BROADCAST_RIGHT_SCALAR || + op->broadcast_kind == LW_X64_FAST_BROADCAST_RIGHT_CHANNEL) { + status = fast_ensure_nhwc(plan, op->input_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + input = fast_nhwc_pointer(plan, op->input_index); + } + output = fast_nhwc_pointer(plan, op->output_index); + if ((input == NULL && node->kind == LW_X64_FAST_NODE_CONTIGUOUS_UNARY) || output == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "elementwise fast buffers are unavailable"); + return LW_STATUS_UNSUPPORTED; + } + if (node->kind == LW_X64_FAST_NODE_CONTIGUOUS_UNARY) { + switch (op->operation) { + case LW_OP_RELU: lw_avx2_relu_contiguous_f32(input, output, elements); break; + case LW_OP_ERF: lw_avx2_erf_f32(input, output, elements); break; + case LW_OP_SQRT: status = lw_scalar_sqrt_f32(input, output, elements); if (status != LW_STATUS_OK) return status; break; + case LW_OP_SIGMOID: status = lw_scalar_sigmoid_f32(input, output, elements); if (status != LW_STATUS_OK) return status; break; + case LW_OP_HARD_SIGMOID: lw_avx2_hard_sigmoid_contiguous_f32(input, output, elements, op->alpha, op->beta); break; + default: return LW_STATUS_UNSUPPORTED; + } + } else { + const float* right = NULL; + const float* full = input; + const float* channel = NULL; + lw_scalar_binary_op operation = op->operation == LW_OP_ADD ? LW_SCALAR_BINARY_ADD : + (op->operation == LW_OP_MUL ? LW_SCALAR_BINARY_MUL : + (op->operation == LW_OP_DIV ? LW_SCALAR_BINARY_DIV : + (op->operation == LW_OP_SUB ? LW_SCALAR_BINARY_SUB : LW_SCALAR_BINARY_POW))); + if (op->broadcast_kind == LW_X64_FAST_BROADCAST_NONE) { + status = fast_ensure_nhwc(plan, op->rhs_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + right = fast_nhwc_pointer(plan, op->rhs_index); + if (input == NULL || right == NULL) return LW_STATUS_UNSUPPORTED; + lw_avx2_binary_contiguous_f32(operation, input, right, output, elements); + } else if (op->broadcast_kind == LW_X64_FAST_BROADCAST_RIGHT_SCALAR) { + float scalar; + status = fast_elementwise_scalar_value(plan, op->rhs_index, graph_input_index, graph_input, &scalar, error); + if (status != LW_STATUS_OK || input == NULL) return status != LW_STATUS_OK ? status : LW_STATUS_UNSUPPORTED; + lw_avx2_binary_right_scalar_f32(operation, input, scalar, output, elements); + } else if (op->broadcast_kind == LW_X64_FAST_BROADCAST_LEFT_SCALAR) { + float scalar; + uint64_t index; + status = fast_elementwise_scalar_value(plan, op->input_index, graph_input_index, graph_input, &scalar, error); + if (status != LW_STATUS_OK) return status; + status = fast_ensure_nhwc(plan, op->rhs_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + right = fast_nhwc_pointer(plan, op->rhs_index); + if (right == NULL) return LW_STATUS_UNSUPPORTED; + for (index = 0u; index < elements; ++index) { + if (operation == LW_SCALAR_BINARY_ADD) output[index] = scalar + right[index]; + else if (operation == LW_SCALAR_BINARY_MUL) output[index] = scalar * right[index]; + else if (operation == LW_SCALAR_BINARY_SUB) output[index] = scalar - right[index]; + else if (operation == LW_SCALAR_BINARY_DIV) output[index] = scalar / right[index]; + else output[index] = powf(scalar, right[index]); + } + } else if (op->broadcast_kind == LW_X64_FAST_BROADCAST_RIGHT_CHANNEL) { + channel = fast_elementwise_constant_f32(plan->session, op->rhs_index); + if (channel == NULL) { + status = fast_ensure_nhwc(plan, op->rhs_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + channel = fast_nhwc_pointer(plan, op->rhs_index); + } + if (full == NULL || channel == NULL) return LW_STATUS_UNSUPPORTED; + lw_avx2_binary_channel_f32(operation, full, channel, output, + elements / op->channels, op->channels, 0); + } else { + channel = fast_elementwise_constant_f32(plan->session, op->input_index); + if (channel == NULL) { + status = fast_ensure_nhwc(plan, op->input_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + channel = fast_nhwc_pointer(plan, op->input_index); + } + status = fast_ensure_nhwc(plan, op->rhs_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + full = fast_nhwc_pointer(plan, op->rhs_index); + if (full == NULL || channel == NULL) return LW_STATUS_UNSUPPORTED; + lw_avx2_binary_channel_f32(operation, full, channel, output, + elements / op->channels, op->channels, 1); + } + } + fast_publish_nhwc(plan, op->output_index); + return LW_STATUS_OK; +} +static lw_status fast_execute_reduce_mean(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* node, + uint32_t graph_input_index, + const float* graph_input, + lw_error* error) { + const lw_x64_fast_spatial* op = &node->data.spatial; + const lw_runtime_tensor* input_tensor = &plan->session->tensors[op->input_index]; + const lw_runtime_tensor* output_tensor = &plan->session->tensors[op->output_index]; + float* input; + float* output; + lw_status status = fast_ensure_nhwc(plan, op->input_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + input = fast_nhwc_pointer(plan, op->input_index); + output = fast_nhwc_pointer(plan, op->output_index); + if (input == NULL || output == NULL) return LW_STATUS_UNSUPPORTED; + lw_avx2_nhwc_reduce_mean_hw_f32(input, output, + (uint32_t)input_tensor->dimensions[0], + (uint32_t)input_tensor->dimensions[2], + (uint32_t)input_tensor->dimensions[3], + (uint32_t)input_tensor->dimensions[1]); + fast_publish_nhwc(plan, op->output_index); + (void)output_tensor; + return LW_STATUS_OK; +} + +static lw_status fast_execute_pool(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* node, + uint32_t graph_input_index, + const float* graph_input, + lw_error* error) { + const lw_x64_fast_spatial* op = &node->data.spatial; + const lw_runtime_tensor* input_tensor = &plan->session->tensors[op->input_index]; + const lw_runtime_tensor* output_tensor = &plan->session->tensors[op->output_index]; + float* input; + float* output; + lw_status status = fast_ensure_nhwc(plan, op->input_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + input = fast_nhwc_pointer(plan, op->input_index); + output = fast_nhwc_pointer(plan, op->output_index); + if (input == NULL || output == NULL) return LW_STATUS_UNSUPPORTED; + lw_avx2_nhwc_pool_f32(input, output, + (uint32_t)input_tensor->dimensions[0], + (uint32_t)input_tensor->dimensions[2], + (uint32_t)input_tensor->dimensions[3], + (uint32_t)output_tensor->dimensions[2], + (uint32_t)output_tensor->dimensions[3], + (uint32_t)input_tensor->dimensions[1], + op->kernel_h, op->kernel_w, op->stride_h, op->stride_w, + op->pad_top, op->pad_left, + op->count_include_pad, op->is_max); + fast_publish_nhwc(plan, op->output_index); + return LW_STATUS_OK; +} + +static lw_status fast_execute_batch_norm(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* node, + uint32_t graph_input_index, + const float* graph_input, + lw_error* error) { + const lw_x64_fast_spatial* op = &node->data.spatial; + const lw_runtime_tensor* input_tensor = &plan->session->tensors[op->input_index]; + float* input; + float* output; + lw_status status = fast_ensure_nhwc(plan, op->input_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + input = fast_nhwc_pointer(plan, op->input_index); + output = fast_nhwc_pointer(plan, op->output_index); + if (input == NULL || output == NULL) return LW_STATUS_UNSUPPORTED; + if (op->affine_mul == NULL || op->affine_add == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "batch normalization affine is unavailable"); + return LW_STATUS_UNSUPPORTED; + } + lw_avx2_nhwc_affine_f32(input, op->affine_mul, op->affine_add, output, + (uint32_t)((uint64_t)input_tensor->dimensions[0] * + input_tensor->dimensions[2] * input_tensor->dimensions[3]), + (uint32_t)input_tensor->dimensions[1]); + fast_publish_nhwc(plan, op->output_index); + return LW_STATUS_OK; +} +static lw_status fast_execute_concat(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* node, + uint32_t graph_input_index, + const float* graph_input, + lw_error* error) { + const lw_x64_fast_spatial* op = &node->data.spatial; + const lw_runtime_tensor* output_tensor = &plan->session->tensors[op->output_index]; + float* output = fast_nhwc_pointer(plan, op->output_index); + uint64_t pixels = (uint64_t)output_tensor->dimensions[0] * output_tensor->dimensions[2] * output_tensor->dimensions[3]; + uint32_t slot; + uint32_t output_channels = 0u; + if (output == NULL) return LW_STATUS_UNSUPPORTED; + for (slot = 0u; slot < op->input_count; ++slot) { + uint32_t input_index = op->input_indices[slot]; + const lw_runtime_tensor* input_tensor = &plan->session->tensors[input_index]; + float* input; + uint64_t pixel; + lw_status status = fast_ensure_nhwc(plan, input_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + input = fast_nhwc_pointer(plan, input_index); + if (input == NULL) return LW_STATUS_UNSUPPORTED; + for (pixel = 0u; pixel < pixels; ++pixel) { + memcpy(output + pixel * output_tensor->dimensions[1] + output_channels, + input + pixel * input_tensor->dimensions[1], + (size_t)op->input_channels[slot] * sizeof(float)); + } + output_channels += op->input_channels[slot]; + } + if (output_channels != (uint32_t)output_tensor->dimensions[1]) return LW_STATUS_INVALID_SHAPE; + fast_publish_nhwc(plan, op->output_index); + return LW_STATUS_OK; +} +static lw_status fast_execute_depthwise(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* node, + uint32_t graph_input_index, + const float* graph_input, + lw_nhwc_depthwise_stats* depthwise_stats, + lw_error* error) { + const lw_x64_fast_conv* conv = &node->data.conv; + lw_nhwc_depthwise_desc desc; + float* input; + float* output; + lw_status status; + status = fast_ensure_nhwc(plan, conv->input_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + input = fast_nhwc_pointer(plan, conv->input_index); + output = fast_nhwc_pointer(plan, conv->output_index); + if (input == NULL || output == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "depthwise fast buffers are unavailable"); + return LW_STATUS_UNSUPPORTED; + } + memset(&desc, 0, sizeof(desc)); + desc.batch = 1u; + desc.channels = conv->input_channels; + desc.input_height = conv->input_height; + desc.input_width = conv->input_width; + desc.output_height = conv->output_height; + desc.output_width = conv->output_width; + desc.kernel_h = conv->kernel_h; + desc.kernel_w = conv->kernel_w; + desc.stride_h = conv->stride_h; + desc.stride_w = conv->stride_w; + desc.pad_top = conv->pad_top; + desc.pad_left = conv->pad_left; + desc.pad_bottom = conv->pad_bottom; + desc.pad_right = conv->pad_right; + status = depthwise_stats == NULL + ? lw_avx2_fma_nhwc_depthwise_f32(input, conv->packed_weights, conv->bias, + output, &desc) + : lw_avx2_fma_nhwc_depthwise_profiled_f32(input, conv->packed_weights, conv->bias, + output, &desc, depthwise_stats); + if (status != LW_STATUS_OK) { + lw_set_error(error, status, "depthwise fast kernel failed"); + return status; + } + fast_publish_nhwc(plan, conv->output_index); + return LW_STATUS_OK; +} +static lw_status fast_execute_gelu(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* node, + uint32_t graph_input_index, + const float* graph_input, + lw_error* error) { + const lw_x64_fast_elementwise* op = &node->data.elementwise; + const lw_runtime_tensor* tensor = &plan->session->tensors[op->output_index]; + float* input; + float* output; + lw_status status; + if (plan->layout.node_layout[node->semantic_begin] == LW_FAST_LAYOUT_NCHW) { + input = (float*)lw_executor_tensor_input_data(plan->session, op->input_index, + graph_input_index, graph_input); + output = lw_executor_tensor_output_data(plan->session, op->output_index); + if (input == NULL || output == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "NCHW GELU buffers are unavailable"); + return LW_STATUS_UNSUPPORTED; + } + lw_avx2_gelu_f32(input, output, tensor->byte_size / sizeof(float)); + fast_publish_nchw(plan, op->output_index); + return LW_STATUS_OK; + } + status = fast_ensure_nhwc(plan, op->input_index, graph_input_index, graph_input, error); + if (status != LW_STATUS_OK) return status; + input = fast_nhwc_pointer(plan, op->input_index); + output = fast_nhwc_pointer(plan, op->output_index); + if (input == NULL || output == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "NHWC GELU buffers are unavailable"); + return LW_STATUS_UNSUPPORTED; + } + lw_avx2_gelu_f32(input, output, tensor->byte_size / sizeof(float)); + fast_publish_nhwc(plan, op->output_index); + return LW_STATUS_OK; +} +static lw_status fast_execute_generic_physical_op(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* op, + uint32_t graph_input_index, + const float* graph_input, + lw_error* error) { + uint32_t consumed = 1u; + lw_status status; + if (plan == NULL || op == NULL) return LW_STATUS_INVALID_ARGUMENT; + status = fast_execute_generic_node(plan, op->semantic_begin, graph_input_index, + graph_input, &consumed, error); + if (status != LW_STATUS_OK) return status; + if (consumed != op->semantic_count) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "generic physical op span mismatch"); + return LW_STATUS_UNSUPPORTED; + } + return LW_STATUS_OK; +} + +static lw_status fast_execute_physical_op(lw_x64_rec_fast_plan* plan, + const lw_x64_physical_op* op, + uint32_t graph_input_index, + const float* graph_input, + lw_nhwc_depthwise_stats* depthwise_stats, + lw_error* error) { + if (plan == NULL || op == NULL) return LW_STATUS_INVALID_ARGUMENT; + switch (op->kind) { + case LW_X64_FAST_NODE_GELU: return fast_execute_gelu(plan, op, graph_input_index, graph_input, error); + case LW_X64_FAST_NODE_BATCH_NORM: return fast_execute_batch_norm(plan, op, graph_input_index, graph_input, error); + case LW_X64_FAST_NODE_CONCAT: return fast_execute_concat(plan, op, graph_input_index, graph_input, error); + case LW_X64_FAST_NODE_REDUCE_MEAN: return fast_execute_reduce_mean(plan, op, graph_input_index, graph_input, error); + case LW_X64_FAST_NODE_POOL: return fast_execute_pool(plan, op, graph_input_index, graph_input, error); + case LW_X64_FAST_NODE_POINTWISE: return fast_execute_pointwise(plan, op, graph_input_index, graph_input, error); + case LW_X64_FAST_NODE_CONTIGUOUS_UNARY: + case LW_X64_FAST_NODE_CONTIGUOUS_BINARY: return fast_execute_elementwise(plan, op, graph_input_index, graph_input, error); + case LW_X64_FAST_NODE_DENSE: return fast_execute_dense(plan, op, graph_input_index, graph_input, error); + case LW_X64_FAST_NODE_DEPTHWISE: return fast_execute_depthwise(plan, op, graph_input_index, graph_input, depthwise_stats, error); + case LW_X64_FAST_NODE_GENERIC: return fast_execute_generic_physical_op(plan, op, graph_input_index, graph_input, error); + default: return LW_STATUS_UNSUPPORTED; + } +} + +static void fast_profile_add(uint64_t* value, uint64_t amount) { + if (value == NULL) return; + if (*value > UINT64_MAX - amount) *value = UINT64_MAX; + else *value += amount; +} +static lw_status fast_execute_backbone(lw_x64_rec_fast_plan* plan, + const float* input, + lw_x64_fast_profile* profile, + lw_nhwc_depthwise_stats* depthwise_stats, + lw_error* error) { + if (plan == NULL) return LW_STATUS_INVALID_ARGUMENT; + for (uint32_t op_index = 0u; op_index < plan->op_count; ++op_index) { + const lw_x64_physical_op* physical = &plan->ops[op_index]; + uint32_t node_index = physical->semantic_begin; + uint64_t started = 0u; + uint64_t finished = 0u; + lw_status status; + if (profile != NULL && profile->clock != NULL) { + started = profile->clock(profile->clock_context); + } + status = fast_execute_physical_op(plan, physical, plan->graph_input_index, input, + profile == NULL ? NULL : depthwise_stats, error); + if (profile != NULL && profile->clock != NULL) { + finished = profile->clock(profile->clock_context); + if (finished < started) finished = started; + fast_profile_add(&profile->total_nanoseconds, finished - started); + if (node_index < LW_X64_FAST_PROFILE_NODE_CAPACITY) { + fast_profile_add(&profile->node_nanoseconds[node_index], finished - started); + fast_profile_add(&profile->node_invocations[node_index], 1u); + } + if (physical->kind < LW_X64_FAST_PROFILE_KIND_CAPACITY) { + fast_profile_add(&profile->kind_nanoseconds[physical->kind], finished - started); + fast_profile_add(&profile->kind_invocations[physical->kind], 1u); + } + } + if (status != LW_STATUS_OK) return status; + } + if (profile != NULL && depthwise_stats != NULL) { + plan->depthwise_x2_invocations = depthwise_stats->x2_invocations; + plan->depthwise_x1_invocations = depthwise_stats->x1_invocations; + } + return LW_STATUS_OK; +} +lw_status lw_x64_rec_fast_run_profiled(lw_x64_rec_fast_plan* plan, const float* input, + uint64_t input_element_count, float* output, + uint64_t output_element_count, + lw_x64_fast_profile* profile, lw_error* error) { + const lw_runtime_tensor* input_tensor; + const lw_runtime_tensor* output_tensor; + const float* graph_output; + lw_x64_fast_profile_clock profile_clock_fn = profile == NULL ? NULL : profile->clock; + void* profile_clock_context = profile == NULL ? NULL : profile->clock_context; + lw_nhwc_depthwise_stats depthwise_stats = { 0u, 0u }; + lw_status status; + if (profile != NULL) { + memset(profile, 0, sizeof(*profile)); + profile->clock = profile_clock_fn; + profile->clock_context = profile_clock_context; + } + if (plan == NULL || plan->session == NULL || plan->session->model == NULL || input == NULL || + output == NULL || plan->graph_input_index >= plan->tensor_count || + plan->graph_output_index >= plan->tensor_count) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "invalid fast execution arguments"); + return LW_STATUS_INVALID_ARGUMENT; + } + input_tensor = &plan->session->tensors[plan->graph_input_index]; + output_tensor = &plan->session->tensors[plan->graph_output_index]; + if (input_tensor->dtype != LW_DTYPE_F32 || output_tensor->dtype != LW_DTYPE_F32 || + input_element_count != fast_element_count(input_tensor) || + output_element_count != fast_element_count(output_tensor)) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "fast execution tensor shape mismatch"); + return LW_STATUS_INVALID_SHAPE; + } + fast_begin_run(plan, LW_X64_FAST_INPUT_NCHW); + status = fast_execute_backbone(plan, input, profile, &depthwise_stats, error); + if (status != LW_STATUS_OK) return status; + if (fast_layout_tracked_tensor(output_tensor)) { + status = fast_ensure_nchw(plan, plan->graph_output_index, error); + if (status != LW_STATUS_OK) return status; + } + graph_output = lw_executor_tensor_input_data(plan->session, plan->graph_output_index, + plan->graph_input_index, input); + if (graph_output == NULL) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "fast executor graph output is unavailable"); + return LW_STATUS_UNSUPPORTED; + } + memcpy(output, graph_output, (size_t)output_tensor->byte_size); + if (profile != NULL) { + profile->conversion_invocations = plan->conversion_count; + profile->conversion_bytes = plan->conversion_bytes; + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} + +lw_status lw_x64_rec_fast_run_prepared_nhwc(lw_x64_rec_fast_plan* plan, + lw_x64_fast_profile* profile, + lw_error* error) { + lw_x64_fast_profile_clock profile_clock_fn = profile == NULL ? NULL : profile->clock; + void* profile_clock_context = profile == NULL ? NULL : profile->clock_context; + lw_nhwc_depthwise_stats depthwise_stats = { 0u, 0u }; + lw_status status; + if (profile != NULL) { + memset(profile, 0, sizeof(*profile)); + profile->clock = profile_clock_fn; + profile->clock_context = profile_clock_context; + } + if (plan == NULL || plan->session == NULL || plan->session->model == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "prepared NHWC fast run requires a valid plan"); + return LW_STATUS_INVALID_ARGUMENT; + } + if (plan->layout.graph_input_direct_nhwc == 0u) { + lw_set_error(error, LW_STATUS_UNSUPPORTED, "prepared NHWC fast run requires a direct NHWC plan"); + return LW_STATUS_UNSUPPORTED; + } + fast_begin_run(plan, LW_X64_FAST_INPUT_NHWC); + status = fast_execute_backbone(plan, NULL, profile, + profile == NULL ? NULL : &depthwise_stats, error); + if (status != LW_STATUS_OK) return status; + if (profile != NULL) { + profile->conversion_invocations = plan->conversion_count; + profile->conversion_bytes = plan->conversion_bytes; + } + lw_set_error(error, LW_STATUS_OK, ""); + return LW_STATUS_OK; +} +lw_status lw_x64_rec_fast_run(lw_x64_rec_fast_plan* plan, const float* input, + uint64_t input_element_count, float* output, + uint64_t output_element_count, lw_error* error) { + return lw_x64_rec_fast_run_profiled(plan, input, input_element_count, output, + output_element_count, NULL, error); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_fast_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_fast_internal.h new file mode 100644 index 00000000..55cb3442 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_fast_internal.h @@ -0,0 +1,253 @@ +#ifndef LW_X64_REC_FAST_INTERNAL_H +#define LW_X64_REC_FAST_INTERNAL_H + +#include "lw_infer.h" +#include "session_internal.h" +#include "layout_planner_internal.h" +#include "../kernels/nhwc_internal.h" + +#include +#include + +#define LW_X64_FAST_OFFSET_NONE UINT64_MAX +#define LW_X64_FAST_OPERATOR_CAPACITY 64u +#define LW_X64_FAST_PROFILE_NODE_CAPACITY 512u +#define LW_X64_FAST_PROFILE_KIND_CAPACITY 16u + +typedef enum lw_x64_fast_input_layout { + LW_X64_FAST_INPUT_NCHW = 0, + LW_X64_FAST_INPUT_NHWC = 1 +} lw_x64_fast_input_layout; + +enum { + LW_X64_FAST_HAVE_NCHW = 1u << 0, + LW_X64_FAST_HAVE_NHWC = 1u << 1 +}; + +typedef enum lw_x64_fast_node_kind { + LW_X64_FAST_NODE_GENERIC = 0, + LW_X64_FAST_NODE_POINTWISE = 1, + LW_X64_FAST_NODE_DENSE = 2, + LW_X64_FAST_NODE_DEPTHWISE = 3, + LW_X64_FAST_NODE_CONTIGUOUS_UNARY = 4, + LW_X64_FAST_NODE_CONTIGUOUS_BINARY = 5, + LW_X64_FAST_NODE_BATCH_NORM = 6, + LW_X64_FAST_NODE_REDUCE_MEAN = 7, + LW_X64_FAST_NODE_POOL = 8, + LW_X64_FAST_NODE_RESIZE = 9, + LW_X64_FAST_NODE_CONCAT = 10, + LW_X64_FAST_NODE_LAYER_NORM = 11, + LW_X64_FAST_NODE_GELU = 12 +} lw_x64_fast_node_kind; + +typedef struct lw_x64_fast_tensor_lifetime { + int32_t birth_op; + int32_t last_use_op; +} lw_x64_fast_tensor_lifetime; + +typedef struct lw_x64_fast_tensor_state { + uint64_t nhwc_offset; + uint8_t available_layouts; + uint8_t neutral; + uint16_t reserved; +} lw_x64_fast_tensor_state; + +typedef struct lw_x64_fast_conv { + uint32_t input_index; + uint32_t output_index; + uint32_t residual_index; + uint32_t weight_index; + uint32_t bias_index; + uint32_t input_channels; + uint32_t output_channels; + uint32_t input_height; + uint32_t input_width; + uint32_t output_height; + uint32_t output_width; + uint32_t kernel_h; + uint32_t kernel_w; + uint32_t stride_h; + uint32_t stride_w; + uint32_t pad_top; + uint32_t pad_left; + uint32_t pad_bottom; + uint32_t pad_right; + uint32_t dense_kc; + float* packed_weights; + uint64_t packed_weight_count; + uint8_t packed_weights_borrowed; + const float* bias; + float* owned_bias; + int32_t* dense_tap_offsets; + int32_t* dense_patch_offsets; + int32_t* dense_input_offsets_k; + int32_t* dense_patch_offsets_k; + uint32_t dense_taps; + uint32_t dense_k_total; + uint32_t dense_patch_width; + uint32_t dense_x_tiles; + uint16_t activation; + uint16_t flags; + uint64_t scratch_bytes; +} lw_x64_fast_conv; + +typedef enum lw_x64_fast_broadcast_kind { + LW_X64_FAST_BROADCAST_NONE = 0, + LW_X64_FAST_BROADCAST_RIGHT_SCALAR = 1, + LW_X64_FAST_BROADCAST_LEFT_SCALAR = 2, + LW_X64_FAST_BROADCAST_RIGHT_CHANNEL = 3, + LW_X64_FAST_BROADCAST_LEFT_CHANNEL = 4 +} lw_x64_fast_broadcast_kind; + +typedef struct lw_x64_fast_elementwise { + uint32_t input_index; + uint32_t rhs_index; + uint32_t output_index; + uint16_t operation; + uint8_t broadcast_kind; + uint8_t reserved; + uint32_t channels; + float alpha; + float beta; +} lw_x64_fast_elementwise; +typedef struct lw_x64_fast_spatial { + uint32_t input_index; + uint32_t output_index; + uint32_t kernel_h; + uint32_t kernel_w; + uint32_t stride_h; + uint32_t stride_w; + uint32_t pad_top; + uint32_t pad_left; + uint32_t pad_bottom; + uint32_t pad_right; + uint8_t count_include_pad; + uint8_t is_max; + uint16_t reserved; + uint32_t input_count; + uint32_t input_indices[LWM_V0_MAX_NODE_INPUTS]; + uint32_t input_channels[LWM_V0_MAX_NODE_INPUTS]; + const float* scale; + const float* bias; + const float* mean; + const float* variance; + float* affine_mul; + float* affine_add; + float epsilon; +} lw_x64_fast_spatial; +typedef struct lw_x64_fast_node { + uint32_t semantic_node_index; + uint16_t kind; + uint16_t semantic_node_count; + uint32_t output_index; + union { + lw_x64_fast_conv conv; + lw_x64_fast_elementwise elementwise; + lw_x64_fast_spatial spatial; + } data; +} lw_x64_fast_node; + +typedef struct lw_x64_physical_op { + uint32_t semantic_begin; + uint16_t semantic_count; + uint16_t kind; + uint32_t output_index; + union { + lw_x64_fast_conv conv; + lw_x64_fast_elementwise elementwise; + lw_x64_fast_spatial spatial; + } data; +} lw_x64_physical_op; +typedef uint64_t (*lw_x64_fast_profile_clock)(void* context); +typedef struct lw_x64_fast_profile { + lw_x64_fast_profile_clock clock; + void* clock_context; + uint64_t node_nanoseconds[LW_X64_FAST_PROFILE_NODE_CAPACITY]; + uint64_t node_invocations[LW_X64_FAST_PROFILE_NODE_CAPACITY]; + uint64_t kind_nanoseconds[LW_X64_FAST_PROFILE_KIND_CAPACITY]; + uint64_t kind_invocations[LW_X64_FAST_PROFILE_KIND_CAPACITY]; + uint64_t conversion_nanoseconds; + uint64_t conversion_invocations; + uint64_t conversion_bytes; + uint64_t total_nanoseconds; +} lw_x64_fast_profile; +typedef struct lw_x64_rec_fast_plan { + lw_session* session; + const struct lw_x64_rec_fast_plan* packed_source; + lw_layout_plan layout; + lw_x64_fast_node* nodes; + lw_x64_physical_op* ops; + uint32_t op_count; + uint32_t op_capacity; + uint32_t node_count; + lw_x64_fast_tensor_state* tensors; + uint32_t* consumer_count; + uint8_t* elided_tensor; + uint32_t elided_tensor_count; + uint32_t tensor_count; + lw_x64_fast_tensor_lifetime* physical_lifetimes; + uint32_t* semantic_to_physical; + uint8_t* nhwc_workspace; + size_t nhwc_workspace_bytes; + uint8_t* scratch; + size_t scratch_bytes; + uint32_t graph_input_index; + uint32_t graph_output_index; + uint32_t fast_node_count; + uint32_t generic_node_count; + uint32_t pointwise_node_count; + uint32_t dense_node_count; + uint32_t depthwise_node_count; + uint32_t binary_node_count; + uint32_t relu_node_count; + uint32_t unary_node_count; + uint32_t reduce_mean_node_count; + uint32_t pool_node_count; + uint32_t concat_node_count; + uint32_t batch_norm_node_count; + uint32_t fused_conv_bn_count; + uint32_t fused_conv_relu_count; + uint32_t fused_conv_bn_relu_count; + uint32_t fused_conv_add_count; + uint32_t fused_conv_add_relu_count; + uint32_t physical_generic_op_count; + uint32_t unsupported_nhwc_node_count; + uint32_t unsupported_nhwc_by_operator[LW_X64_FAST_OPERATOR_CAPACITY]; + uint64_t conversion_count; + uint64_t conversion_bytes; + uint64_t depthwise_x2_invocations; + uint64_t depthwise_x1_invocations; +} lw_x64_rec_fast_plan; + +lw_status lw_x64_rec_fast_plan_create(lw_session* session, + lw_x64_rec_fast_plan** out_plan, + lw_error* error); +lw_status lw_x64_rec_fast_plan_create_shared(lw_session* session, + const lw_x64_rec_fast_plan* packed_source, + lw_x64_rec_fast_plan** out_plan, + lw_error* error); +void lw_x64_rec_fast_plan_free(lw_x64_rec_fast_plan* plan); +lw_status lw_x64_rec_fast_run_profiled(lw_x64_rec_fast_plan* plan, + const float* input, uint64_t input_element_count, + float* output, uint64_t output_element_count, + lw_x64_fast_profile* profile, lw_error* error); +float* lw_x64_rec_fast_graph_input_nhwc(lw_x64_rec_fast_plan* plan, + uint64_t* element_count); +lw_status lw_x64_rec_fast_run_prepared_nhwc(lw_x64_rec_fast_plan* plan, + lw_x64_fast_profile* profile, + lw_error* error); +lw_status lw_x64_rec_fast_run(lw_x64_rec_fast_plan* plan, + const float* input, + uint64_t input_element_count, + float* output, + uint64_t output_element_count, + lw_error* error); + +void lw_x64_fast_nchw_to_nhwc(const float* source, float* destination, + uint32_t batch, uint32_t channels, + uint32_t height, uint32_t width); +void lw_x64_fast_nhwc_to_nchw(const float* source, float* destination, + uint32_t batch, uint32_t channels, + uint32_t height, uint32_t width); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_fast_plan.c b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_fast_plan.c new file mode 100644 index 00000000..b510846d --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/runtime/x64_rec_fast_plan.c @@ -0,0 +1,1364 @@ +#include "x64_rec_fast_internal.h" + +#include "lwm_read.h" +#include "operator_internal.h" + +#include +#include +#include +#include + +static size_t fast_align64(size_t value) { + return (value + 63u) & ~(size_t)63u; +} + + + + +static int fast_tensor_eligible(const lw_runtime_tensor* tensor) { + return tensor != NULL && (tensor->flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u && + tensor->dtype == LW_DTYPE_F32 && tensor->rank == 4u && + tensor->dimensions[0] > 0 && tensor->dimensions[1] > 0 && + tensor->dimensions[2] > 0 && tensor->dimensions[3] > 0; +} + +static const float* fast_constant_f32(const lw_session* session, uint32_t tensor_index) { + const lw_runtime_tensor* tensor; + const uint8_t* disk; + uint64_t data_offset; + if (session == NULL || session->model == NULL || tensor_index >= session->model->info.tensor_count) { + return NULL; + } + tensor = &session->tensors[tensor_index]; + if (tensor->dtype != LW_DTYPE_F32 || + (tensor->flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u) return NULL; + disk = session->model->bytes + (size_t)session->model->tensor_offset + + (size_t)tensor_index * LWM_V0_TENSOR_SIZE; + data_offset = lwm_read_u64(disk + 48u); + if (data_offset > session->model->byte_count || + tensor->byte_size > session->model->byte_count - data_offset) return NULL; + return (const float*)(const void*)(session->model->bytes + (size_t)data_offset); +} + +static int fast_same_shape4(const lw_runtime_tensor* left, const lw_runtime_tensor* right) { + if (!fast_tensor_eligible(left) || !fast_tensor_eligible(right) || left->rank != right->rank) return 0; + for (uint32_t index = 0u; index < 4u; ++index) { + if (left->dimensions[index] != right->dimensions[index]) return 0; + } + return 1; +} + +static uint64_t fast_tensor_elements(const lw_runtime_tensor* tensor) { + return tensor == NULL || tensor->dtype != LW_DTYPE_F32 || tensor->byte_size % sizeof(float) != 0u + ? 0u : tensor->byte_size / sizeof(float); +} + +static float fast_read_f32(const uint8_t* bytes) { + float value = 0.0f; + if (bytes != NULL) memcpy(&value, bytes, sizeof(value)); + return value; +} + +static int fast_channel_broadcast(const lw_session* session, uint32_t tensor_index, + const lw_runtime_tensor* full) { + const lw_runtime_tensor* tensor; + uint64_t elements; + if (session == NULL || full == NULL || tensor_index >= session->model->info.tensor_count) return 0; + tensor = &session->tensors[tensor_index]; + elements = fast_tensor_elements(tensor); + if (elements == 1u) return 1; + if ((tensor->flags & LWM_V0_TENSOR_FLAG_CONSTANT) != 0u && + elements == (uint64_t)(uint32_t)full->dimensions[1]) return 1; + return tensor->rank == 4u && tensor->dtype == LW_DTYPE_F32 && + tensor->dimensions[1] == full->dimensions[1] && tensor->dimensions[2] == 1 && + tensor->dimensions[3] == 1 && + (tensor->dimensions[0] == 1); +} + +static int fast_classify_binary(const lw_session* session, uint32_t left_index, + uint32_t right_index, uint32_t output_index, + uint8_t* broadcast_kind) { + const lw_runtime_tensor* left; + const lw_runtime_tensor* right; + const lw_runtime_tensor* output; + if (session == NULL || broadcast_kind == NULL || left_index >= session->model->info.tensor_count || + right_index >= session->model->info.tensor_count || output_index >= session->model->info.tensor_count) return 0; + left = &session->tensors[left_index]; right = &session->tensors[right_index]; + output = &session->tensors[output_index]; + if (!fast_tensor_eligible(output)) return 0; + if (fast_same_shape4(left, output) && fast_same_shape4(right, output)) { + *broadcast_kind = LW_X64_FAST_BROADCAST_NONE; return 1; + } + if (fast_same_shape4(left, output) && fast_tensor_elements(right) == 1u) { + *broadcast_kind = LW_X64_FAST_BROADCAST_RIGHT_SCALAR; return 1; + } + if (fast_same_shape4(right, output) && fast_tensor_elements(left) == 1u) { + *broadcast_kind = LW_X64_FAST_BROADCAST_LEFT_SCALAR; return 1; + } + if (fast_same_shape4(left, output) && fast_channel_broadcast(session, right_index, output)) { + *broadcast_kind = LW_X64_FAST_BROADCAST_RIGHT_CHANNEL; return 1; + } + if (fast_same_shape4(right, output) && fast_channel_broadcast(session, left_index, output)) { + *broadcast_kind = LW_X64_FAST_BROADCAST_LEFT_CHANNEL; return 1; + } + return 0; +} + +static int fast_gelu_temporaries_are_private(const lw_x64_rec_fast_plan* plan, + uint32_t node_index, + const lw_fused_gelu_match* match) { + uint32_t index; + if (plan == NULL || match == NULL || plan->session == NULL || + node_index > plan->node_count || plan->node_count - node_index < 5u) return 0; + for (index = 0u; index < 4u; ++index) { + uint32_t tensor_index = match->outputs[index]; + if (tensor_index >= plan->tensor_count || plan->consumer_count == NULL || + plan->consumer_count[tensor_index] != 1u || + plan->session->tensors[tensor_index].last_use_node != + (int32_t)(node_index + index + 1u)) return 0; + } + return 1; +} +static void fast_elide_tensor(lw_x64_rec_fast_plan* plan, uint32_t tensor_index); +static int fast_prepare_gelu(lw_x64_rec_fast_plan* plan, uint32_t node_index, + lw_x64_fast_node* fast_node) { + lw_fused_gelu_match match; + const lw_runtime_tensor* input; + const lw_runtime_tensor* output; + if (plan == NULL || fast_node == NULL || node_index >= plan->node_count || + !lw_match_fused_gelu(plan->session, node_index, &match)) return 0; + if (!fast_gelu_temporaries_are_private(plan, node_index, &match)) return 0; + for (uint32_t index = 1u; index < 5u; ++index) { + if (plan->layout.node_layout[node_index + index] != + plan->layout.node_layout[node_index]) return 0; + } + if (match.inputs[0][0] >= plan->tensor_count || match.outputs[4] >= plan->tensor_count) return 0; + input = &plan->session->tensors[match.inputs[0][0]]; + output = &plan->session->tensors[match.outputs[4]]; + if (!fast_same_shape4(input, output)) return 0; + memset(&fast_node->data.elementwise, 0, sizeof(fast_node->data.elementwise)); + fast_node->data.elementwise.input_index = match.inputs[0][0]; + fast_node->data.elementwise.output_index = match.outputs[4]; + fast_node->data.elementwise.operation = LW_OP_ERF; + for (uint32_t index = 0u; index < 4u; ++index) fast_elide_tensor(plan, match.outputs[index]); + fast_node->kind = LW_X64_FAST_NODE_GELU; + fast_node->semantic_node_count = 5u; + fast_node->output_index = match.outputs[4]; + ++plan->unary_node_count; + return 1; +} +static int fast_prepare_elementwise(lw_x64_rec_fast_plan* plan, uint32_t node_index, + lw_x64_fast_node* fast_node) { + const uint8_t* node; + const uint8_t* params; + uint16_t operation; + uint16_t input_count; + uint32_t input_index; + uint32_t rhs_index = UINT32_MAX; + uint32_t output_index; + const lw_runtime_tensor* input; + const lw_runtime_tensor* output; + int is_binary; + int is_unary; + if (plan == NULL || fast_node == NULL || node_index >= plan->node_count || + plan->layout.node_layout[node_index] != LW_FAST_LAYOUT_NHWC) return 0; + node = plan->session->model->bytes + (size_t)plan->session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + operation = lwm_read_u16(node); input_count = lwm_read_u16(node + 2u); + input_index = lwm_read_u32(node + 8u); output_index = lwm_read_u32(node + 40u); + is_binary = operation == LW_OP_ADD || operation == LW_OP_MUL || operation == LW_OP_DIV || + operation == LW_OP_SUB || operation == LW_OP_POW; + is_unary = operation == LW_OP_RELU || operation == LW_OP_SIGMOID || operation == LW_OP_SQRT || + operation == LW_OP_ERF || operation == LW_OP_HARD_SIGMOID; + if ((!is_binary && !is_unary) || input_index >= plan->tensor_count || output_index >= plan->tensor_count) return 0; + if ((is_unary && input_count != 1u) || (is_binary && input_count != 2u)) return 0; + input = &plan->session->tensors[input_index]; output = &plan->session->tensors[output_index]; + if (!fast_same_shape4(input, output)) return 0; + memset(&fast_node->data.elementwise, 0, sizeof(fast_node->data.elementwise)); + fast_node->data.elementwise.input_index = input_index; + fast_node->data.elementwise.output_index = output_index; + fast_node->data.elementwise.operation = operation; + fast_node->data.elementwise.channels = (uint32_t)output->dimensions[1]; + if (is_binary) { + uint8_t broadcast_kind; + rhs_index = lwm_read_u32(node + 12u); + if (!fast_classify_binary(plan->session, input_index, rhs_index, output_index, &broadcast_kind)) return 0; + fast_node->data.elementwise.rhs_index = rhs_index; + fast_node->data.elementwise.broadcast_kind = broadcast_kind; + fast_node->kind = LW_X64_FAST_NODE_CONTIGUOUS_BINARY; + ++plan->binary_node_count; + } else { + fast_node->kind = LW_X64_FAST_NODE_CONTIGUOUS_UNARY; + if (operation == LW_OP_HARD_SIGMOID) { + params = plan->session->model->bytes + (size_t)lwm_read_u64(node + 56u); + fast_node->data.elementwise.alpha = fast_read_f32(params + 4u); + fast_node->data.elementwise.beta = fast_read_f32(params + 8u); + } + ++plan->unary_node_count; + if (operation == LW_OP_RELU) ++plan->relu_node_count; + } + return 1; +} + +static void fast_record_nhwc_blocker(lw_x64_rec_fast_plan* plan, const uint8_t* node) { + uint32_t operation; + if (plan == NULL || node == NULL) return; + operation = (uint32_t)lwm_read_u16(node); + if (plan->unsupported_nhwc_node_count != UINT32_MAX) ++plan->unsupported_nhwc_node_count; + if (operation < LW_X64_FAST_OPERATOR_CAPACITY && + plan->unsupported_nhwc_by_operator[operation] != UINT32_MAX) { + ++plan->unsupported_nhwc_by_operator[operation]; + } +} + +static int fast_prepare_reduce_mean(lw_x64_rec_fast_plan* plan, uint32_t node_index, + lw_x64_fast_node* fast_node) { + const uint8_t* node; + const uint8_t* params; + const lw_runtime_tensor* input; + const lw_runtime_tensor* output; + int32_t axis0; + int32_t axis1; + if (plan == NULL || fast_node == NULL || node_index >= plan->node_count || + plan->layout.node_layout[node_index] != LW_FAST_LAYOUT_NHWC) return 0; + node = plan->session->model->bytes + (size_t)plan->session->model->node_offset + (size_t)node_index * LWM_V0_NODE_SIZE; + params = plan->session->model->bytes + (size_t)lwm_read_u64(node + 56u); + if (lwm_read_u16(node) != LW_OP_REDUCE_MEAN || lwm_read_u16(node + 2u) != 1u || + lwm_read_u16(node + 4u) != 1u || lwm_read_u16(params + 2u) != 2u || lwm_read_u32(params + 4u) == 0u) return 0; + input = &plan->session->tensors[lwm_read_u32(node + 8u)]; output = &plan->session->tensors[lwm_read_u32(node + 40u)]; + axis0 = lwm_read_i32(params + 12u); axis1 = lwm_read_i32(params + 16u); + if (axis0 < 0) axis0 += 4; + if (axis1 < 0) axis1 += 4; + if (!fast_tensor_eligible(input) || !fast_tensor_eligible(output) || + !((axis0 == 2 && axis1 == 3) || (axis0 == 3 && axis1 == 2)) || + output->dimensions[0] != input->dimensions[0] || output->dimensions[1] != input->dimensions[1] || + output->dimensions[2] != 1 || output->dimensions[3] != 1) return 0; + memset(&fast_node->data.spatial, 0, sizeof(fast_node->data.spatial)); + fast_node->data.spatial.input_index = lwm_read_u32(node + 8u); + fast_node->data.spatial.output_index = lwm_read_u32(node + 40u); + fast_node->kind = LW_X64_FAST_NODE_REDUCE_MEAN; + ++plan->reduce_mean_node_count; + return 1; +} + +static int fast_prepare_pool(lw_x64_rec_fast_plan* plan, uint32_t node_index, + lw_x64_fast_node* fast_node) { + const uint8_t* node; + const uint8_t* params; + const lw_runtime_tensor* input; + const lw_runtime_tensor* output; + if (plan == NULL || fast_node == NULL || node_index >= plan->node_count || + plan->layout.node_layout[node_index] != LW_FAST_LAYOUT_NHWC) return 0; + node = plan->session->model->bytes + (size_t)plan->session->model->node_offset + (size_t)node_index * LWM_V0_NODE_SIZE; + params = plan->session->model->bytes + (size_t)lwm_read_u64(node + 56u); + if ((lwm_read_u16(node) != LW_OP_AVERAGE_POOL && lwm_read_u16(node) != LW_OP_MAX_POOL) || + lwm_read_u16(node + 2u) != 1u || lwm_read_u16(node + 4u) != 1u) return 0; + input = &plan->session->tensors[lwm_read_u32(node + 8u)]; output = &plan->session->tensors[lwm_read_u32(node + 40u)]; + if (!fast_tensor_eligible(input) || !fast_tensor_eligible(output)) return 0; + if (lwm_read_i32(params + 8u) <= 0 || lwm_read_i32(params + 12u) <= 0 || lwm_read_i32(params + 16u) <= 0 || lwm_read_i32(params + 20u) <= 0) return 0; + memset(&fast_node->data.spatial, 0, sizeof(fast_node->data.spatial)); + fast_node->data.spatial.input_index = lwm_read_u32(node + 8u); fast_node->data.spatial.output_index = lwm_read_u32(node + 40u); + fast_node->data.spatial.kernel_h = (uint32_t)lwm_read_i32(params + 8u); fast_node->data.spatial.kernel_w = (uint32_t)lwm_read_i32(params + 12u); + fast_node->data.spatial.stride_h = (uint32_t)lwm_read_i32(params + 16u); fast_node->data.spatial.stride_w = (uint32_t)lwm_read_i32(params + 20u); + fast_node->data.spatial.pad_top = (uint32_t)lwm_read_i32(params + 24u); fast_node->data.spatial.pad_left = (uint32_t)lwm_read_i32(params + 28u); + fast_node->data.spatial.pad_bottom = (uint32_t)lwm_read_i32(params + 32u); fast_node->data.spatial.pad_right = (uint32_t)lwm_read_i32(params + 36u); + fast_node->data.spatial.count_include_pad = (uint8_t)lwm_read_u32(params + 44u); + fast_node->data.spatial.is_max = (uint8_t)(lwm_read_u16(node) == LW_OP_MAX_POOL); + fast_node->kind = LW_X64_FAST_NODE_POOL; + ++plan->pool_node_count; + return 1; +} +static int fast_prepare_concat(lw_x64_rec_fast_plan* plan, uint32_t node_index, + lw_x64_fast_node* fast_node) { + const uint8_t* node; + const uint8_t* params; + const lw_runtime_tensor* output; + uint32_t count; + uint32_t slot; + int32_t axis; + if (plan == NULL || fast_node == NULL || node_index >= plan->node_count || + plan->layout.node_layout[node_index] != LW_FAST_LAYOUT_NHWC) return 0; + node = plan->session->model->bytes + (size_t)plan->session->model->node_offset + (size_t)node_index * LWM_V0_NODE_SIZE; + params = plan->session->model->bytes + (size_t)lwm_read_u64(node + 56u); + count = lwm_read_u16(node + 2u); + axis = lwm_read_i32(params + 4u); if (axis < 0) axis += 4; + if (lwm_read_u16(node) != LW_OP_CONCAT || count == 0u || count > LWM_V0_MAX_NODE_INPUTS || + lwm_read_u16(node + 4u) != 1u || axis != 1) return 0; + output = &plan->session->tensors[lwm_read_u32(node + 40u)]; + if (!fast_tensor_eligible(output)) return 0; + memset(&fast_node->data.spatial, 0, sizeof(fast_node->data.spatial)); + fast_node->data.spatial.output_index = lwm_read_u32(node + 40u); + fast_node->data.spatial.input_count = count; + for (slot = 0u; slot < count; ++slot) { + uint32_t input_index = lwm_read_u32(node + 8u + (size_t)slot * sizeof(uint32_t)); + const lw_runtime_tensor* input; + if (input_index >= plan->tensor_count || !fast_tensor_eligible(input = &plan->session->tensors[input_index]) || + input->dimensions[0] != output->dimensions[0] || input->dimensions[2] != output->dimensions[2] || + input->dimensions[3] != output->dimensions[3]) return 0; + fast_node->data.spatial.input_indices[slot] = input_index; + fast_node->data.spatial.input_channels[slot] = (uint32_t)input->dimensions[1]; + } + fast_node->kind = LW_X64_FAST_NODE_CONCAT; + ++plan->concat_node_count; + return 1; +} +static int fast_prepare_batch_norm(lw_x64_rec_fast_plan* plan, uint32_t node_index, + lw_x64_fast_node* fast_node) { + const uint8_t* node; + const uint8_t* params; + const lw_runtime_tensor* input; + const lw_runtime_tensor* output; + uint32_t input_index; + uint32_t output_index; + uint32_t c; + const float* scale; + const float* bias; + const float* mean; + const float* variance; + if (plan == NULL || fast_node == NULL || node_index >= plan->node_count || + plan->layout.node_layout[node_index] != LW_FAST_LAYOUT_NHWC) return 0; + node = plan->session->model->bytes + (size_t)plan->session->model->node_offset + (size_t)node_index * LWM_V0_NODE_SIZE; + params = plan->session->model->bytes + (size_t)lwm_read_u64(node + 56u); + if (lwm_read_u16(node) != LW_OP_BATCH_NORMALIZATION || lwm_read_u16(node + 2u) < 5u || lwm_read_u16(node + 4u) != 1u) return 0; + input_index = lwm_read_u32(node + 8u); output_index = lwm_read_u32(node + 40u); + if (input_index >= plan->tensor_count || output_index >= plan->tensor_count) return 0; + input = &plan->session->tensors[input_index]; output = &plan->session->tensors[output_index]; + if (!fast_tensor_eligible(input) || !fast_tensor_eligible(output) || + input->dimensions[0] != output->dimensions[0] || input->dimensions[1] != output->dimensions[1] || + input->dimensions[2] != output->dimensions[2] || input->dimensions[3] != output->dimensions[3]) return 0; + scale = fast_constant_f32(plan->session, lwm_read_u32(node + 12u)); + bias = fast_constant_f32(plan->session, lwm_read_u32(node + 16u)); + mean = fast_constant_f32(plan->session, lwm_read_u32(node + 20u)); + variance = fast_constant_f32(plan->session, lwm_read_u32(node + 24u)); + c = (uint32_t)input->dimensions[1]; + if (scale == NULL || bias == NULL || mean == NULL || variance == NULL || + plan->session->tensors[lwm_read_u32(node + 12u)].byte_size < (uint64_t)c * sizeof(float) || + plan->session->tensors[lwm_read_u32(node + 16u)].byte_size < (uint64_t)c * sizeof(float) || + plan->session->tensors[lwm_read_u32(node + 20u)].byte_size < (uint64_t)c * sizeof(float) || + plan->session->tensors[lwm_read_u32(node + 24u)].byte_size < (uint64_t)c * sizeof(float)) return 0; + memset(&fast_node->data.spatial, 0, sizeof(fast_node->data.spatial)); + fast_node->data.spatial.input_index = input_index; fast_node->data.spatial.output_index = output_index; + fast_node->data.spatial.scale = scale; fast_node->data.spatial.bias = bias; + fast_node->data.spatial.mean = mean; fast_node->data.spatial.variance = variance; + fast_node->data.spatial.epsilon = fast_read_f32(params + 4u); + fast_node->data.spatial.affine_mul = (float*)malloc((size_t)c * sizeof(float)); + fast_node->data.spatial.affine_add = (float*)malloc((size_t)c * sizeof(float)); + if (fast_node->data.spatial.affine_mul == NULL || fast_node->data.spatial.affine_add == NULL) { + free(fast_node->data.spatial.affine_mul); + free(fast_node->data.spatial.affine_add); + fast_node->data.spatial.affine_mul = NULL; + fast_node->data.spatial.affine_add = NULL; + return 0; + } + for (uint32_t channel = 0u; channel < c; ++channel) { + float variance_epsilon = variance[channel] + fast_node->data.spatial.epsilon; + float multiplier; + float additive; + if (!(variance_epsilon > 0.0f) || !isfinite(variance_epsilon)) { + free(fast_node->data.spatial.affine_mul); + free(fast_node->data.spatial.affine_add); + fast_node->data.spatial.affine_mul = NULL; + fast_node->data.spatial.affine_add = NULL; + return 0; + } + multiplier = scale[channel] / sqrtf(variance_epsilon); + additive = bias[channel] - mean[channel] * multiplier; + if (!isfinite(multiplier) || !isfinite(additive)) { + free(fast_node->data.spatial.affine_mul); + free(fast_node->data.spatial.affine_add); + fast_node->data.spatial.affine_mul = NULL; + fast_node->data.spatial.affine_add = NULL; + return 0; + } + fast_node->data.spatial.affine_mul[channel] = multiplier; + fast_node->data.spatial.affine_add[channel] = additive; + } + fast_node->kind = LW_X64_FAST_NODE_BATCH_NORM; + ++plan->batch_norm_node_count; + return 1; +} +static int fast_conv_output_matches(const lw_runtime_tensor* input, const lw_runtime_tensor* output, + int32_t kh, int32_t kw, int32_t sh, int32_t sw, + int32_t pt, int32_t pl, int32_t pb, int32_t pr) { + int64_t numerator_h; + int64_t numerator_w; + if (input == NULL || output == NULL || kh <= 0 || kw <= 0 || sh <= 0 || sw <= 0 || + pt < 0 || pl < 0 || pb < 0 || pr < 0) return 0; + numerator_h = (int64_t)input->dimensions[2] + pt + pb - kh; + numerator_w = (int64_t)input->dimensions[3] + pl + pr - kw; + if (numerator_h < 0 || numerator_w < 0) return 0; + return numerator_h / sh + 1 == output->dimensions[2] && + numerator_w / sw + 1 == output->dimensions[3]; +} +static int fast_prepare_dense_offsets(lw_x64_fast_conv* conv) { + uint64_t taps64; + uint64_t k_total64; + uint64_t patch_width64; + uint64_t x_tiles64; + if (conv == NULL || conv->input_channels == 0u || conv->kernel_h == 0u || + conv->kernel_w == 0u || conv->stride_w == 0u || + (uint64_t)conv->kernel_h * conv->kernel_w > UINT32_MAX || + (uint64_t)conv->input_channels * conv->kernel_h * conv->kernel_w > UINT32_MAX) { + return 0; + } + taps64 = (uint64_t)conv->kernel_h * conv->kernel_w; + k_total64 = taps64 * conv->input_channels; + patch_width64 = ((uint64_t)LW_NHWC_PIXEL_TILE - 1u) * conv->stride_w + conv->kernel_w; + x_tiles64 = ((uint64_t)conv->output_width + LW_NHWC_PIXEL_TILE - 1u) / LW_NHWC_PIXEL_TILE; + if (patch_width64 > UINT32_MAX || x_tiles64 > UINT32_MAX || + k_total64 > SIZE_MAX / sizeof(int32_t)) return 0; + conv->dense_taps = (uint32_t)taps64; + conv->dense_k_total = (uint32_t)k_total64; + conv->dense_patch_width = (uint32_t)patch_width64; + conv->dense_x_tiles = (uint32_t)x_tiles64; + conv->dense_input_offsets_k = (int32_t*)malloc((size_t)k_total64 * sizeof(int32_t)); + conv->dense_patch_offsets_k = (int32_t*)malloc((size_t)k_total64 * sizeof(int32_t)); + if (conv->dense_input_offsets_k == NULL || conv->dense_patch_offsets_k == NULL) { + free(conv->dense_input_offsets_k); + free(conv->dense_patch_offsets_k); + conv->dense_input_offsets_k = NULL; + conv->dense_patch_offsets_k = NULL; + return 0; + } + for (uint32_t ky = 0u; ky < conv->kernel_h; ++ky) { + for (uint32_t kx = 0u; kx < conv->kernel_w; ++kx) { + for (uint32_t channel = 0u; channel < conv->input_channels; ++channel) { + uint64_t tap = (uint64_t)ky * conv->kernel_w + kx; + uint64_t k = (uint64_t)channel * taps64 + tap; + uint64_t input_offset = (((uint64_t)ky * conv->input_width + kx) * + conv->input_channels) + channel; + uint64_t patch_offset = (((uint64_t)ky * patch_width64 + kx) * + conv->input_channels) + channel; + if (input_offset > INT32_MAX || patch_offset > INT32_MAX) { + free(conv->dense_input_offsets_k); + free(conv->dense_patch_offsets_k); + conv->dense_input_offsets_k = NULL; + conv->dense_patch_offsets_k = NULL; + return 0; + } + conv->dense_input_offsets_k[k] = (int32_t)input_offset; + conv->dense_patch_offsets_k[k] = (int32_t)patch_offset; + } + } + } + return 1; +} +static const lw_x64_physical_op* fast_packed_source_op(const lw_x64_rec_fast_plan* plan, + uint32_t node_index) { + uint32_t index; + if (plan == NULL || plan->packed_source == NULL) return NULL; + for (index = 0u; index < plan->packed_source->op_count; ++index) { + const lw_x64_physical_op* op = &plan->packed_source->ops[index]; + if (op->semantic_begin == node_index) return op; + } + return NULL; +} + +static int fast_prepare_conv(lw_x64_rec_fast_plan* plan, uint32_t node_index, + lw_x64_fast_node* fast_node) { + lw_session* session; + const uint8_t* node; + const uint8_t* params; + const lw_runtime_tensor* input; + const lw_runtime_tensor* weight; + const lw_runtime_tensor* output; + lw_x64_fast_conv* conv; + uint32_t input_index; + uint32_t weight_index; + uint32_t bias_index; + uint32_t group; + int32_t kh, kw, sh, sw, dh, dw, pt, pl, pb, pr; + uint64_t packed_count; + uint64_t scratch_bytes; + int is_depthwise; + const lw_x64_physical_op* packed_op; + if (plan == NULL || fast_node == NULL || node_index >= plan->node_count || + plan->layout.node_layout[node_index] != LW_FAST_LAYOUT_NHWC) return 0; + session = plan->session; + node = session->model->bytes + (size_t)session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + if (lwm_read_u16(node) != LW_OP_CONV || lwm_read_u16(node + 2u) < 2u || + lwm_read_u16(node + 2u) > 3u || lwm_read_u16(node + 4u) != 1u) return 0; + params = session->model->bytes + (size_t)lwm_read_u64(node + 56u); + input_index = lwm_read_u32(node + 8u); + weight_index = lwm_read_u32(node + 12u); + bias_index = lwm_read_u16(node + 2u) >= 3u ? lwm_read_u32(node + 16u) : UINT32_MAX; + if (input_index >= plan->tensor_count || weight_index >= plan->tensor_count || + (bias_index != UINT32_MAX && bias_index >= plan->tensor_count)) return 0; + input = &session->tensors[input_index]; + weight = &session->tensors[weight_index]; + if (lwm_read_u32(node + 40u) >= plan->tensor_count) return 0; + output = &session->tensors[lwm_read_u32(node + 40u)]; + group = lwm_read_u32(params + 4u); + kh = lwm_read_i32(params + 8u); kw = lwm_read_i32(params + 12u); + sh = lwm_read_i32(params + 16u); sw = lwm_read_i32(params + 20u); + dh = lwm_read_i32(params + 24u); dw = lwm_read_i32(params + 28u); + pt = lwm_read_i32(params + 32u); pl = lwm_read_i32(params + 36u); + pb = lwm_read_i32(params + 40u); pr = lwm_read_i32(params + 44u); + is_depthwise = group == (uint32_t)input->dimensions[1] && group > 1u && + output->dimensions[1] == input->dimensions[1] && + (input->dimensions[1] & 7) == 0u && + weight->dimensions[1] == 1; + if ((!is_depthwise && group != 1u) || kh <= 0 || kw <= 0 || sh <= 0 || sw <= 0 || dh != 1 || dw != 1 || + pt < 0 || pl < 0 || pb < 0 || pr < 0 || !fast_conv_output_matches(input, output, kh, kw, sh, sw, pt, pl, pb, pr) || !fast_tensor_eligible(input) || + !fast_tensor_eligible(output) || weight->dtype != LW_DTYPE_F32 || weight->rank != 4u || + (weight->flags & LWM_V0_TENSOR_FLAG_CONSTANT) == 0u || + weight->dimensions[0] != output->dimensions[1] || + (is_depthwise ? weight->dimensions[1] != 1 : weight->dimensions[1] != input->dimensions[1]) || + weight->dimensions[2] != kh || + weight->dimensions[3] != kw || + (is_depthwise ? ((uint32_t)output->dimensions[1] < 8u || ((uint32_t)output->dimensions[1] & 7u) != 0u) : + ((uint32_t)output->dimensions[1] < 8u))) return 0; + conv = &fast_node->data.conv; + memset(conv, 0, sizeof(*conv)); + conv->residual_index = UINT32_MAX; + conv->activation = LW_NHWC_ACT_NONE; + conv->input_index = input_index; conv->output_index = lwm_read_u32(node + 40u); + conv->weight_index = weight_index; conv->bias_index = bias_index; + conv->input_channels = (uint32_t)input->dimensions[1]; + conv->output_channels = (uint32_t)output->dimensions[1]; + conv->input_height = (uint32_t)input->dimensions[2]; + conv->input_width = (uint32_t)input->dimensions[3]; + conv->output_height = (uint32_t)output->dimensions[2]; + conv->output_width = (uint32_t)output->dimensions[3]; + conv->kernel_h = (uint32_t)kh; conv->kernel_w = (uint32_t)kw; + conv->stride_h = (uint32_t)sh; conv->stride_w = (uint32_t)sw; + conv->pad_top = (uint32_t)pt; conv->pad_left = (uint32_t)pl; + conv->pad_bottom = (uint32_t)pb; conv->pad_right = (uint32_t)pr; + conv->dense_kc = LW_NHWC_DENSE_KC; + conv->bias = bias_index == UINT32_MAX ? NULL : fast_constant_f32(session, bias_index); + if (bias_index != UINT32_MAX && conv->bias == NULL) return 0; + if (fast_constant_f32(session, weight_index) == NULL || + (is_depthwise ? !lw_nhwc_depthwise_packed_weight_count(conv->input_channels, + conv->kernel_h, conv->kernel_w, &packed_count) : + !lw_nhwc_dense_packed_weight_count(conv->input_channels, conv->output_channels, + conv->kernel_h, conv->kernel_w, &packed_count)) || + packed_count > SIZE_MAX / sizeof(float)) return 0; + packed_op = fast_packed_source_op(plan, node_index); + if (plan->packed_source != NULL) { + const lw_x64_fast_conv* source; + if (packed_op == NULL || (packed_op->kind != LW_X64_FAST_NODE_POINTWISE && + packed_op->kind != LW_X64_FAST_NODE_DENSE && + packed_op->kind != LW_X64_FAST_NODE_DEPTHWISE)) return 0; + source = &packed_op->data.conv; + if (source->weight_index != weight_index || source->input_channels != conv->input_channels || + source->output_channels != conv->output_channels || source->kernel_h != conv->kernel_h || + source->kernel_w != conv->kernel_w || source->packed_weight_count != packed_count || + source->packed_weights == NULL) return 0; + conv->packed_weights = source->packed_weights; + conv->packed_weights_borrowed = 1u; + } else { + conv->packed_weights = (float*)malloc((size_t)packed_count * sizeof(float)); + if (conv->packed_weights == NULL) return 0; + } + conv->packed_weight_count = packed_count; + if (is_depthwise) { + if (!conv->packed_weights_borrowed) { + lw_pack_nhwc_depthwise_f32(fast_constant_f32(session, weight_index), + conv->input_channels, conv->kernel_h, conv->kernel_w, conv->packed_weights); + } + fast_node->kind = LW_X64_FAST_NODE_DEPTHWISE; + ++plan->depthwise_node_count; + return 1; + } + if (!conv->packed_weights_borrowed) { + lw_pack_nhwc_dense_f32(fast_constant_f32(session, weight_index), conv->input_channels, + conv->output_channels, conv->kernel_h, conv->kernel_w, + conv->packed_weights); + } + if (kh == 1 && kw == 1 && sh == 1 && sw == 1 && pt == 0 && pl == 0 && pb == 0 && pr == 0) { + fast_node->kind = LW_X64_FAST_NODE_POINTWISE; + ++plan->pointwise_node_count; + } else { + lw_nhwc_dense_desc desc; + memset(&desc, 0, sizeof(desc)); + desc.batch = (uint32_t)input->dimensions[0]; + desc.input_channels = conv->input_channels; desc.input_height = conv->input_height; + desc.input_width = conv->input_width; desc.output_channels = conv->output_channels; + desc.output_height = conv->output_height; desc.output_width = conv->output_width; + desc.kernel_h = conv->kernel_h; desc.kernel_w = conv->kernel_w; + desc.stride_h = conv->stride_h; desc.stride_w = conv->stride_w; + desc.pad_top = conv->pad_top; desc.pad_left = conv->pad_left; + desc.pad_bottom = conv->pad_bottom; desc.pad_right = conv->pad_right; + desc.dense_kc = conv->dense_kc; + if (!fast_prepare_dense_offsets(conv) || + !lw_nhwc_dense_prepared_scratch_bytes(&desc, &scratch_bytes) || scratch_bytes > SIZE_MAX) { + free(conv->dense_input_offsets_k); + free(conv->dense_patch_offsets_k); + conv->dense_input_offsets_k = NULL; + conv->dense_patch_offsets_k = NULL; + if (!conv->packed_weights_borrowed) free(conv->packed_weights); + conv->packed_weights = NULL; return 0; + } + conv->scratch_bytes = scratch_bytes; + if ((size_t)scratch_bytes > plan->scratch_bytes) plan->scratch_bytes = (size_t)scratch_bytes; + fast_node->kind = LW_X64_FAST_NODE_DENSE; + ++plan->dense_node_count; + } + return 1; +} +static lw_status fast_init_tensor_states(lw_x64_rec_fast_plan* plan, lw_error* error) { + if (plan == NULL || plan->session == NULL || plan->session->model == NULL) { + return LW_STATUS_INVALID_ARGUMENT; + } + plan->tensor_count = plan->session->model->info.tensor_count; + plan->tensors = (lw_x64_fast_tensor_state*)calloc(plan->tensor_count, sizeof(*plan->tensors)); + if (plan->tensors == NULL && plan->tensor_count != 0u) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate fast tensor states"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (uint32_t index = 0u; index < plan->tensor_count; ++index) { + lw_x64_fast_tensor_state* state = &plan->tensors[index]; + state->nhwc_offset = LW_X64_FAST_OFFSET_NONE; + state->neutral = (uint8_t)lw_layout_tensor_is_neutral(plan->session, index); + state->available_layouts = state->neutral + ? (uint8_t)(LW_X64_FAST_HAVE_NCHW | LW_X64_FAST_HAVE_NHWC) + : LW_X64_FAST_HAVE_NCHW; + } + return LW_STATUS_OK; +} + +static lw_status fast_build_consumer_counts(lw_x64_rec_fast_plan* plan, lw_error* error) { + uint32_t node_index; + if (plan == NULL || plan->session == NULL || plan->session->model == NULL) { + return LW_STATUS_INVALID_ARGUMENT; + } + plan->consumer_count = (uint32_t*)calloc(plan->tensor_count, sizeof(*plan->consumer_count)); + plan->elided_tensor = (uint8_t*)calloc(plan->tensor_count, sizeof(*plan->elided_tensor)); + if ((plan->consumer_count == NULL || plan->elided_tensor == NULL) && plan->tensor_count != 0u) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate fast consumer metadata"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (node_index = 0u; node_index < plan->node_count; ++node_index) { + const uint8_t* node = plan->session->model->bytes + + (size_t)plan->session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + uint32_t input_count = lwm_read_u16(node + 2u); + for (uint32_t input_slot = 0u; input_slot < input_count; ++input_slot) { + uint32_t tensor_index = lwm_read_u32(node + 8u + + (size_t)input_slot * sizeof(uint32_t)); + if (tensor_index >= plan->tensor_count) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "fast consumer tensor index out of bounds"); + return LW_STATUS_OUT_OF_BOUNDS; + } + if (plan->consumer_count[tensor_index] == UINT32_MAX) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "fast consumer count overflow"); + return LW_STATUS_OUT_OF_BOUNDS; + } + ++plan->consumer_count[tensor_index]; + } + } + return LW_STATUS_OK; +} + +static void fast_elide_tensor(lw_x64_rec_fast_plan* plan, uint32_t tensor_index) { + if (plan != NULL && plan->elided_tensor != NULL && tensor_index < plan->tensor_count && + plan->elided_tensor[tensor_index] == 0u) { + plan->elided_tensor[tensor_index] = 1u; + if (plan->elided_tensor_count != UINT32_MAX) ++plan->elided_tensor_count; + } +} +typedef struct fast_workspace_interval { + uint32_t tensor_index; + int32_t birth; + int32_t last_use; + size_t bytes; +} fast_workspace_interval; + +typedef struct fast_workspace_slot { + size_t offset; + size_t bytes; + int32_t last_use; +} fast_workspace_slot; + +static int fast_tensor_needs_nhwc(const lw_x64_rec_fast_plan* plan, uint32_t tensor_index) { + const lw_runtime_tensor* tensor; + if (plan == NULL || tensor_index >= plan->tensor_count) return 0; + tensor = &plan->session->tensors[tensor_index]; + if (plan->elided_tensor != NULL && plan->elided_tensor[tensor_index] != 0u) return 0; + return fast_tensor_eligible(tensor) && plan->tensors[tensor_index].neutral == 0u && + (plan->layout.tensor_available_layouts[tensor_index] & LW_LAYOUT_AVAILABLE_NHWC) != 0u; +} + + +static int fast_tensor_needs_nhwc(const lw_x64_rec_fast_plan* plan, uint32_t tensor_index); + +static lw_status fast_build_semantic_to_physical(lw_x64_rec_fast_plan* plan, lw_error* error) { + if (plan == NULL) return LW_STATUS_INVALID_ARGUMENT; + plan->semantic_to_physical = (uint32_t*)malloc((size_t)plan->node_count * sizeof(*plan->semantic_to_physical)); + if (plan->semantic_to_physical == NULL && plan->node_count != 0u) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate semantic-to-physical map"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (uint32_t index = 0u; index < plan->node_count; ++index) { + plan->semantic_to_physical[index] = UINT32_MAX; + } + for (uint32_t op_index = 0u; op_index < plan->op_count; ++op_index) { + const lw_x64_physical_op* op = &plan->ops[op_index]; + uint32_t begin = op->semantic_begin; + uint32_t count = op->semantic_count == 0u ? 1u : op->semantic_count; + if (begin >= plan->node_count || count > plan->node_count - begin) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "invalid x64 physical semantic span"); + return LW_STATUS_INVALID_SHAPE; + } + for (uint32_t offset = 0u; offset < count; ++offset) { + uint32_t semantic = begin + offset; + if (plan->semantic_to_physical[semantic] != UINT32_MAX) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "overlapping x64 physical semantic spans"); + return LW_STATUS_INVALID_SHAPE; + } + plan->semantic_to_physical[semantic] = op_index; + } + } + for (uint32_t index = 0u; index < plan->node_count; ++index) { + if (plan->semantic_to_physical[index] == UINT32_MAX) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "semantic node is missing from x64 physical plan"); + return LW_STATUS_INVALID_SHAPE; + } + } + return LW_STATUS_OK; +} + +static lw_status fast_build_physical_lifetimes(lw_x64_rec_fast_plan* plan, lw_error* error) { + if (plan == NULL || plan->session == NULL || plan->semantic_to_physical == NULL) { + return LW_STATUS_INVALID_ARGUMENT; + } + plan->physical_lifetimes = (lw_x64_fast_tensor_lifetime*)calloc( + plan->tensor_count, sizeof(*plan->physical_lifetimes)); + if (plan->physical_lifetimes == NULL && plan->tensor_count != 0u) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate physical tensor lifetimes"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (uint32_t index = 0u; index < plan->tensor_count; ++index) { + const lw_runtime_tensor* tensor = &plan->session->tensors[index]; + lw_x64_fast_tensor_lifetime* lifetime = &plan->physical_lifetimes[index]; + lifetime->birth_op = 0; + lifetime->last_use_op = 0; + if (tensor->birth_node >= 0) { + uint32_t semantic = (uint32_t)tensor->birth_node; + if (semantic >= plan->node_count || plan->semantic_to_physical[semantic] >= plan->op_count) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "tensor birth semantic node is not in physical plan"); + return LW_STATUS_INVALID_SHAPE; + } + lifetime->birth_op = (int32_t)plan->semantic_to_physical[semantic]; + } + if (tensor->last_use_node >= 0) { + uint32_t semantic = (uint32_t)tensor->last_use_node; + if (semantic == plan->node_count && plan->op_count != 0u) { + /* The session planner uses node_count as the graph-output sentinel. */ + lifetime->last_use_op = (int32_t)(plan->op_count - 1u); + } else { + if (semantic >= plan->node_count || plan->semantic_to_physical[semantic] >= plan->op_count) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "tensor last-use semantic node is not in physical plan"); + return LW_STATUS_INVALID_SHAPE; + } + lifetime->last_use_op = (int32_t)plan->semantic_to_physical[semantic]; + } + } + if (lifetime->last_use_op < lifetime->birth_op) lifetime->last_use_op = lifetime->birth_op; + } + return LW_STATUS_OK; +} + +static int fast_lifetimes_overlap(const lw_x64_fast_tensor_lifetime* left, + const lw_x64_fast_tensor_lifetime* right) { + return left != NULL && right != NULL && + left->birth_op <= right->last_use_op && + right->birth_op <= left->last_use_op; +} + +static int fast_ranges_overlap(uint64_t left_offset, uint64_t left_bytes, + uint64_t right_offset, uint64_t right_bytes) { + if (left_bytes == 0u || right_bytes == 0u) return 0; + return left_offset < right_offset + ? left_bytes > right_offset - left_offset + : right_bytes > left_offset - right_offset; +} + +static int fast_validate_workspace_aliasing(const lw_x64_rec_fast_plan* plan) { + if (plan == NULL || plan->physical_lifetimes == NULL) return 1; + for (uint32_t left = 0u; left < plan->tensor_count; ++left) { + if (!fast_tensor_needs_nhwc(plan, left)) continue; + for (uint32_t right = left + 1u; right < plan->tensor_count; ++right) { + if (!fast_tensor_needs_nhwc(plan, right) || + !fast_lifetimes_overlap(&plan->physical_lifetimes[left], + &plan->physical_lifetimes[right])) continue; + if (fast_ranges_overlap(plan->tensors[left].nhwc_offset, + plan->session->tensors[left].byte_size, + plan->tensors[right].nhwc_offset, + plan->session->tensors[right].byte_size)) return 0; + } + } + return 1; +} +static lw_status fast_allocate_nhwc_workspace(lw_x64_rec_fast_plan* plan, lw_error* error) { + fast_workspace_interval* intervals = NULL; + fast_workspace_slot* slots = NULL; + uint32_t interval_count = 0u; + uint32_t slot_count = 0u; + size_t workspace_end = 0u; + if (plan == NULL) return LW_STATUS_INVALID_ARGUMENT; + intervals = (fast_workspace_interval*)calloc(plan->tensor_count, sizeof(*intervals)); + slots = (fast_workspace_slot*)calloc(plan->tensor_count, sizeof(*slots)); + if ((intervals == NULL || slots == NULL) && plan->tensor_count != 0u) { + free(intervals); free(slots); + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate fast workspace intervals"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (uint32_t index = 0u; index < plan->tensor_count; ++index) { + const lw_runtime_tensor* tensor = &plan->session->tensors[index]; + if (!fast_tensor_needs_nhwc(plan, index)) continue; + if (tensor->byte_size > SIZE_MAX - 63u) { + free(intervals); free(slots); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "fast NHWC tensor size overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + intervals[interval_count].tensor_index = index; + if (plan->physical_lifetimes == NULL) { + free(intervals); free(slots); + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "physical tensor lifetimes are unavailable"); + return LW_STATUS_INVALID_SHAPE; + } + intervals[interval_count].birth = plan->physical_lifetimes[index].birth_op; + intervals[interval_count].last_use = plan->physical_lifetimes[index].last_use_op; + intervals[interval_count].bytes = fast_align64((size_t)tensor->byte_size); + ++interval_count; + } + for (uint32_t index = 1u; index < interval_count; ++index) { + fast_workspace_interval value = intervals[index]; + uint32_t position = index; + while (position > 0u && intervals[position - 1u].birth > value.birth) { + intervals[position] = intervals[position - 1u]; + --position; + } + intervals[position] = value; + } + for (uint32_t index = 0u; index < interval_count; ++index) { + fast_workspace_interval* interval = &intervals[index]; + uint32_t best_slot = UINT32_MAX; + size_t best_waste = SIZE_MAX; + for (uint32_t slot_index = 0u; slot_index < slot_count; ++slot_index) { + fast_workspace_slot* slot = &slots[slot_index]; + if (slot->last_use < interval->birth && slot->bytes >= interval->bytes) { + size_t waste = slot->bytes - interval->bytes; + if (waste < best_waste) { best_waste = waste; best_slot = slot_index; } + } + } + if (best_slot != UINT32_MAX) { + plan->tensors[interval->tensor_index].nhwc_offset = (uint64_t)slots[best_slot].offset; + slots[best_slot].last_use = interval->last_use; + } else { + workspace_end = fast_align64(workspace_end); + if (interval->bytes > SIZE_MAX - workspace_end) { + free(intervals); free(slots); + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "fast NHWC workspace size overflows"); + return LW_STATUS_OUT_OF_BOUNDS; + } + plan->tensors[interval->tensor_index].nhwc_offset = (uint64_t)workspace_end; + slots[slot_count].offset = workspace_end; + slots[slot_count].bytes = interval->bytes; + slots[slot_count].last_use = interval->last_use; + ++slot_count; + workspace_end += interval->bytes; + } + } + free(intervals); free(slots); + plan->nhwc_workspace_bytes = fast_align64(workspace_end); + if (plan->nhwc_workspace_bytes != 0u) { + plan->nhwc_workspace = (uint8_t*)malloc(plan->nhwc_workspace_bytes); + if (plan->nhwc_workspace == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate fast NHWC workspace"); + return LW_STATUS_OUT_OF_MEMORY; + } + } + return LW_STATUS_OK; +} + +static int fast_conv_epilogue_supported(uint16_t kind) { + return kind == LW_X64_FAST_NODE_POINTWISE || kind == LW_X64_FAST_NODE_DENSE; +} + +static int fast_conv_bn_fold_supported(uint16_t kind) { + return kind == LW_X64_FAST_NODE_POINTWISE || + kind == LW_X64_FAST_NODE_DENSE || + kind == LW_X64_FAST_NODE_DEPTHWISE; +} + +static int fast_scale_conv_weights(lw_x64_fast_conv* conv, uint16_t kind, + const float* scale, const float* bn_bias, + const float* mean, const float* variance, + float epsilon) { + uint32_t c; + float* folded_bias; + float* mul; + if (conv == NULL || scale == NULL || bn_bias == NULL || mean == NULL || variance == NULL || + conv->packed_weights == NULL || conv->output_channels == 0u) return 0; + folded_bias = (float*)malloc((size_t)conv->output_channels * sizeof(float)); + mul = (float*)malloc((size_t)conv->output_channels * sizeof(float)); + if (folded_bias == NULL || mul == NULL) { + free(folded_bias); + free(mul); + return 0; + } + for (c = 0u; c < conv->output_channels; ++c) { + float variance_epsilon = variance[c] + epsilon; + float old_bias = conv->bias == NULL ? 0.0f : conv->bias[c]; + if (!(variance_epsilon > 0.0f) || !isfinite(variance_epsilon)) { + free(folded_bias); + free(mul); + return 0; + } + mul[c] = scale[c] / sqrtf(variance_epsilon); + if (!isfinite(mul[c])) { + free(folded_bias); + free(mul); + return 0; + } + folded_bias[c] = old_bias * mul[c] + bn_bias[c] - mean[c] * mul[c]; + if (!isfinite(folded_bias[c])) { + free(folded_bias); + free(mul); + return 0; + } + } + if (kind == LW_X64_FAST_NODE_DEPTHWISE) { + uint32_t taps = conv->kernel_h * conv->kernel_w; + uint32_t blocks = (conv->output_channels + LW_NHWC_DEPTHWISE_BLOCK - 1u) / + LW_NHWC_DEPTHWISE_BLOCK; + for (uint32_t block = 0u; block < blocks; ++block) { + for (uint32_t tap = 0u; tap < taps; ++tap) { + float* weights = conv->packed_weights + + ((size_t)block * taps + tap) * LW_NHWC_DEPTHWISE_BLOCK; + for (uint32_t lane = 0u; lane < LW_NHWC_DEPTHWISE_BLOCK; ++lane) { + uint32_t channel = block * LW_NHWC_DEPTHWISE_BLOCK + lane; + if (channel < conv->output_channels) weights[lane] *= mul[channel]; + } + } + } + } else { + uint64_t k_total = (uint64_t)conv->input_channels * conv->kernel_h * conv->kernel_w; + uint32_t blocks = conv->output_channels / LW_NHWC_OC_BLOCK; + for (uint32_t block = 0u; block < blocks; ++block) { + for (uint64_t k = 0u; k < k_total; ++k) { + float* weights = conv->packed_weights + + ((size_t)block * k_total + (size_t)k) * LW_NHWC_OC_BLOCK; + for (uint32_t lane = 0u; lane < LW_NHWC_OC_BLOCK; ++lane) { + uint32_t channel = block * LW_NHWC_OC_BLOCK + lane; + weights[lane] *= mul[channel]; + } + } + } + } + free(mul); + free(conv->owned_bias); + conv->owned_bias = folded_bias; + conv->bias = folded_bias; + return 1; +} +static void fast_set_physical_output(lw_x64_physical_op* op, uint32_t tensor_index) { + if (op == NULL) return; + op->output_index = tensor_index; + if (op->kind == LW_X64_FAST_NODE_POINTWISE || + op->kind == LW_X64_FAST_NODE_DENSE || + op->kind == LW_X64_FAST_NODE_DEPTHWISE) { + op->data.conv.output_index = tensor_index; + } +} + +static int fast_fuse_conv_bn(lw_x64_rec_fast_plan* plan, lw_x64_physical_op* op, + uint32_t bn_node_index) { + const uint8_t* node; + const uint8_t* params; + uint32_t conv_output; + uint32_t output_index; + uint32_t scale_index; + uint32_t bias_index; + uint32_t mean_index; + uint32_t variance_index; + const lw_runtime_tensor* conv_tensor; + const lw_runtime_tensor* output_tensor; + uint32_t channels; + const float* scale; + const float* bias; + const float* mean; + const float* variance; + if (plan == NULL || op == NULL || !fast_conv_bn_fold_supported(op->kind) || + bn_node_index >= plan->node_count || op->data.conv.output_index >= plan->tensor_count || + plan->consumer_count == NULL || plan->consumer_count[op->data.conv.output_index] != 1u) return 0; + node = plan->session->model->bytes + (size_t)plan->session->model->node_offset + + (size_t)bn_node_index * LWM_V0_NODE_SIZE; + if (lwm_read_u16(node) != LW_OP_BATCH_NORMALIZATION || lwm_read_u16(node + 2u) < 5u || + lwm_read_u16(node + 4u) != 1u || lwm_read_u32(node + 8u) != op->data.conv.output_index) return 0; + conv_output = op->data.conv.output_index; + output_index = lwm_read_u32(node + 40u); + if (output_index >= plan->tensor_count) return 0; + conv_tensor = &plan->session->tensors[conv_output]; + output_tensor = &plan->session->tensors[output_index]; + if (!fast_same_shape4(conv_tensor, output_tensor) || plan->layout.node_layout[bn_node_index] != + plan->layout.node_layout[op->semantic_begin]) return 0; + scale_index = lwm_read_u32(node + 12u); + bias_index = lwm_read_u32(node + 16u); + mean_index = lwm_read_u32(node + 20u); + variance_index = lwm_read_u32(node + 24u); + channels = (uint32_t)conv_tensor->dimensions[1]; + scale = fast_constant_f32(plan->session, scale_index); + bias = fast_constant_f32(plan->session, bias_index); + mean = fast_constant_f32(plan->session, mean_index); + variance = fast_constant_f32(plan->session, variance_index); + params = plan->session->model->bytes + (size_t)lwm_read_u64(node + 56u); + if (scale == NULL || bias == NULL || mean == NULL || variance == NULL || + scale_index >= plan->tensor_count || bias_index >= plan->tensor_count || + mean_index >= plan->tensor_count || variance_index >= plan->tensor_count || + plan->session->tensors[scale_index].byte_size < (uint64_t)channels * sizeof(float) || + plan->session->tensors[bias_index].byte_size < (uint64_t)channels * sizeof(float) || + plan->session->tensors[mean_index].byte_size < (uint64_t)channels * sizeof(float) || + plan->session->tensors[variance_index].byte_size < (uint64_t)channels * sizeof(float)) return 0; + if (op->data.conv.packed_weights_borrowed) { + const lw_x64_physical_op* source_op = fast_packed_source_op(plan, op->semantic_begin); + if (source_op == NULL || source_op->data.conv.owned_bias == NULL || + source_op->semantic_count < 2u) return 0; + op->data.conv.bias = source_op->data.conv.bias; + } else if (!fast_scale_conv_weights(&op->data.conv, op->kind, scale, bias, mean, + variance, fast_read_f32(params + 4u))) { + return 0; + } + fast_set_physical_output(op, output_index); + ++op->semantic_count; + fast_elide_tensor(plan, conv_output); + return 1; +} + +static int fast_fuse_conv_add(lw_x64_rec_fast_plan* plan, lw_x64_physical_op* op, + uint32_t add_node_index) { + const uint8_t* node; + uint32_t left; + uint32_t right; + uint32_t residual; + uint32_t output_index; + uint32_t conv_output; + if (plan == NULL || op == NULL || !fast_conv_epilogue_supported(op->kind) || + add_node_index >= plan->node_count || op->data.conv.output_index >= plan->tensor_count || + plan->consumer_count == NULL || plan->consumer_count[op->data.conv.output_index] != 1u) return 0; + node = plan->session->model->bytes + (size_t)plan->session->model->node_offset + + (size_t)add_node_index * LWM_V0_NODE_SIZE; + if (lwm_read_u16(node) != LW_OP_ADD || lwm_read_u16(node + 2u) != 2u || + lwm_read_u16(node + 4u) != 1u) return 0; + conv_output = op->data.conv.output_index; + left = lwm_read_u32(node + 8u); + right = lwm_read_u32(node + 12u); + if (left == op->data.conv.output_index) residual = right; + else if (right == op->data.conv.output_index) residual = left; + else return 0; + output_index = lwm_read_u32(node + 40u); + if (output_index >= plan->tensor_count || residual >= plan->tensor_count || + plan->consumer_count[output_index] != 1u || + (plan->session->tensors[residual].flags & LWM_V0_TENSOR_FLAG_CONSTANT) != 0u || + !fast_same_shape4(&plan->session->tensors[op->data.conv.output_index], + &plan->session->tensors[residual]) || + !fast_same_shape4(&plan->session->tensors[op->data.conv.output_index], + &plan->session->tensors[output_index]) || + plan->layout.node_layout[add_node_index] != plan->layout.node_layout[op->semantic_begin]) return 0; + op->data.conv.residual_index = residual; + fast_set_physical_output(op, output_index); + ++op->semantic_count; + fast_elide_tensor(plan, conv_output); + return 1; +} + +static int fast_fuse_conv_relu(lw_x64_rec_fast_plan* plan, lw_x64_physical_op* op, + uint32_t relu_node_index) { + const uint8_t* node; + uint32_t input_index; + uint32_t output_index; + if (plan == NULL || op == NULL || !fast_conv_epilogue_supported(op->kind) || + relu_node_index >= plan->node_count || op->data.conv.output_index >= plan->tensor_count || + plan->consumer_count == NULL || plan->consumer_count[op->data.conv.output_index] != 1u) return 0; + node = plan->session->model->bytes + (size_t)plan->session->model->node_offset + + (size_t)relu_node_index * LWM_V0_NODE_SIZE; + if (lwm_read_u16(node) != LW_OP_RELU || lwm_read_u16(node + 2u) != 1u || + lwm_read_u16(node + 4u) != 1u) return 0; + input_index = lwm_read_u32(node + 8u); + output_index = lwm_read_u32(node + 40u); + if (input_index != op->data.conv.output_index || output_index >= plan->tensor_count || + !fast_same_shape4(&plan->session->tensors[input_index], &plan->session->tensors[output_index]) || + plan->layout.node_layout[relu_node_index] != plan->layout.node_layout[op->semantic_begin]) return 0; + op->data.conv.activation = LW_NHWC_ACT_RELU; + fast_set_physical_output(op, output_index); + ++op->semantic_count; + fast_elide_tensor(plan, input_index); + return 1; +} +static int fast_physical_op_owns_conv(uint16_t kind) { + return kind == LW_X64_FAST_NODE_POINTWISE || + kind == LW_X64_FAST_NODE_DENSE || + kind == LW_X64_FAST_NODE_DEPTHWISE; +} + +static int fast_physical_op_owns_spatial(uint16_t kind) { + return kind == LW_X64_FAST_NODE_BATCH_NORM || + kind == LW_X64_FAST_NODE_REDUCE_MEAN || + kind == LW_X64_FAST_NODE_POOL || + kind == LW_X64_FAST_NODE_CONCAT; +} + +static void fast_release_spatial(lw_x64_fast_spatial* spatial) { + if (spatial == NULL) return; + free(spatial->affine_mul); + free(spatial->affine_add); + spatial->affine_mul = NULL; + spatial->affine_add = NULL; +} + +static void fast_move_node_to_physical(const lw_x64_fast_node* node, + lw_x64_physical_op* op) { + if (node == NULL || op == NULL) return; + memset(op, 0, sizeof(*op)); + op->semantic_begin = node->semantic_node_index; + op->semantic_count = node->semantic_node_count == 0u ? 1u : node->semantic_node_count; + op->kind = node->kind; + op->output_index = node->output_index; + memcpy(&op->data, &node->data, sizeof(op->data)); +} + +static void fast_release_conv(lw_x64_fast_conv* conv) { + if (conv == NULL) return; + if (!conv->packed_weights_borrowed) free(conv->packed_weights); + free(conv->owned_bias); + free(conv->dense_tap_offsets); + free(conv->dense_patch_offsets); + free(conv->dense_input_offsets_k); + free(conv->dense_patch_offsets_k); + conv->packed_weights = NULL; + conv->owned_bias = NULL; + conv->dense_tap_offsets = NULL; + conv->dense_patch_offsets = NULL; + conv->dense_input_offsets_k = NULL; + conv->dense_patch_offsets_k = NULL; +} + +static lw_status fast_compile_physical_ops(lw_x64_rec_fast_plan* plan, lw_error* error) { + uint32_t node_index; + if (plan == NULL) return LW_STATUS_INVALID_ARGUMENT; + plan->op_capacity = plan->node_count; + plan->ops = (lw_x64_physical_op*)calloc(plan->op_capacity, sizeof(*plan->ops)); + if (plan->ops == NULL && plan->op_capacity != 0u) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate x64 physical ops"); + return LW_STATUS_OUT_OF_MEMORY; + } + for (node_index = 0u; node_index < plan->node_count;) { + lw_x64_fast_node* node = &plan->nodes[node_index]; + lw_x64_physical_op* op; + uint32_t consumed = node->semantic_node_count == 0u ? 1u : node->semantic_node_count; + if (plan->op_count >= plan->op_capacity) { + lw_set_error(error, LW_STATUS_OUT_OF_BOUNDS, "x64 physical op count overflow"); + return LW_STATUS_OUT_OF_BOUNDS; + } + op = &plan->ops[plan->op_count++]; + fast_move_node_to_physical(node, op); + if (op->kind == LW_X64_FAST_NODE_GENERIC && plan->physical_generic_op_count != UINT32_MAX) ++plan->physical_generic_op_count; + if (fast_physical_op_owns_conv(op->kind)) { + node->data.conv.packed_weights = NULL; + node->data.conv.owned_bias = NULL; + node->data.conv.dense_tap_offsets = NULL; + node->data.conv.dense_patch_offsets = NULL; + node->data.conv.dense_input_offsets_k = NULL; + node->data.conv.dense_patch_offsets_k = NULL; + { + uint32_t start = op->semantic_begin; + int fused_bn = 0; + int fused_add = 0; + int fused_relu = 0; + if (start + 1u < plan->node_count) { + fused_bn = fast_fuse_conv_bn(plan, op, start + 1u); + if (!fused_bn) fused_add = fast_fuse_conv_add(plan, op, start + 1u); + } + if (start + op->semantic_count < plan->node_count) { + fused_relu = fast_fuse_conv_relu(plan, op, start + op->semantic_count); + } + if (fused_bn && fused_relu) { + if (plan->fused_conv_bn_relu_count != UINT32_MAX) ++plan->fused_conv_bn_relu_count; + } else if (fused_bn) { + if (plan->fused_conv_bn_count != UINT32_MAX) ++plan->fused_conv_bn_count; + } else if (fused_add && fused_relu) { + if (plan->fused_conv_add_relu_count != UINT32_MAX) ++plan->fused_conv_add_relu_count; + } else if (fused_add) { + if (plan->fused_conv_add_count != UINT32_MAX) ++plan->fused_conv_add_count; + } else if (fused_relu) { + if (plan->fused_conv_relu_count != UINT32_MAX) ++plan->fused_conv_relu_count; + } + } + } else if (fast_physical_op_owns_spatial(op->kind)) { + node->data.spatial.affine_mul = NULL; + node->data.spatial.affine_add = NULL; + } + consumed = op->semantic_count; + if (consumed > plan->node_count - node_index) consumed = 1u; + node_index += consumed; + } + return LW_STATUS_OK; +} +lw_status lw_x64_rec_fast_plan_create_shared(lw_session* session, + const lw_x64_rec_fast_plan* packed_source, + lw_x64_rec_fast_plan** out_plan, + lw_error* error) { + lw_x64_rec_fast_plan* plan; + lw_layout_planner_options options; + lw_status status; + if (out_plan == NULL || session == NULL || session->model == NULL) { + lw_set_error(error, LW_STATUS_INVALID_ARGUMENT, "fast plan requires a session"); + return LW_STATUS_INVALID_ARGUMENT; + } + *out_plan = NULL; + plan = (lw_x64_rec_fast_plan*)calloc(1u, sizeof(*plan)); + if (plan == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate fast plan"); + return LW_STATUS_OUT_OF_MEMORY; + } + plan->session = session; + plan->packed_source = packed_source; + plan->node_count = session->model->info.node_count; + plan->nodes = (lw_x64_fast_node*)calloc(plan->node_count, sizeof(*plan->nodes)); + if (plan->nodes == NULL && plan->node_count != 0u) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate fast nodes"); + lw_x64_rec_fast_plan_free(plan); + return LW_STATUS_OUT_OF_MEMORY; + } + options.allow_direct_nhwc_graph_input = 1u; + status = lw_layout_plan_build(session, &options, &plan->layout, error); + if (status != LW_STATUS_OK) { + lw_x64_rec_fast_plan_free(plan); + return status; + } + status = fast_init_tensor_states(plan, error); + if (status != LW_STATUS_OK) { + lw_x64_rec_fast_plan_free(plan); + return status; + } + status = fast_build_consumer_counts(plan, error); + if (status != LW_STATUS_OK) { + lw_x64_rec_fast_plan_free(plan); + return status; + } + plan->graph_input_index = lwm_read_u32(session->model->bytes + + (size_t)session->model->input_offset); + plan->graph_output_index = lwm_read_u32(session->model->bytes + + (size_t)session->model->output_offset); + for (uint32_t node_index = 0u; node_index < plan->node_count; ++node_index) { + const uint8_t* node = session->model->bytes + (size_t)session->model->node_offset + + (size_t)node_index * LWM_V0_NODE_SIZE; + plan->nodes[node_index].semantic_node_index = node_index; + plan->nodes[node_index].kind = LW_X64_FAST_NODE_GENERIC; + plan->nodes[node_index].semantic_node_count = 1u; + plan->nodes[node_index].output_index = lwm_read_u32(node + 40u); + if (fast_prepare_gelu(plan, node_index, &plan->nodes[node_index]) || + fast_prepare_elementwise(plan, node_index, &plan->nodes[node_index]) || + fast_prepare_reduce_mean(plan, node_index, &plan->nodes[node_index]) || + fast_prepare_pool(plan, node_index, &plan->nodes[node_index]) || + fast_prepare_concat(plan, node_index, &plan->nodes[node_index]) || + fast_prepare_batch_norm(plan, node_index, &plan->nodes[node_index]) || + fast_prepare_conv(plan, node_index, &plan->nodes[node_index])) { + ++plan->fast_node_count; + } else if (plan->layout.node_layout[node_index] == LW_FAST_LAYOUT_NHWC) { + fast_record_nhwc_blocker(plan, node); + } + } + status = fast_compile_physical_ops(plan, error); + if (status != LW_STATUS_OK) { + lw_x64_rec_fast_plan_free(plan); + return status; + } + if (packed_source != NULL) { + uint32_t index; + for (index = 0u; index < plan->op_count; ++index) { + const lw_x64_physical_op* op = &plan->ops[index]; + const lw_x64_physical_op* source_op; + if (!fast_physical_op_owns_conv(op->kind) || + !op->data.conv.packed_weights_borrowed) continue; + source_op = fast_packed_source_op(plan, op->semantic_begin); + if (source_op == NULL || source_op->kind != op->kind || + source_op->semantic_count != op->semantic_count || + source_op->data.conv.packed_weights != op->data.conv.packed_weights || + source_op->data.conv.bias != op->data.conv.bias) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, + "shared x64 REC weights do not match the physical op"); + lw_x64_rec_fast_plan_free(plan); + return LW_STATUS_INVALID_SHAPE; + } + } + } + status = fast_build_semantic_to_physical(plan, error); + if (status != LW_STATUS_OK) { + lw_x64_rec_fast_plan_free(plan); + return status; + } + status = fast_build_physical_lifetimes(plan, error); + if (status != LW_STATUS_OK) { + lw_x64_rec_fast_plan_free(plan); + return status; + } + status = fast_allocate_nhwc_workspace(plan, error); + if (status != LW_STATUS_OK) { + lw_x64_rec_fast_plan_free(plan); + return status; + } + if (!fast_validate_workspace_aliasing(plan)) { + lw_set_error(error, LW_STATUS_INVALID_SHAPE, "x64 physical workspace alias detected"); + lw_x64_rec_fast_plan_free(plan); + return LW_STATUS_INVALID_SHAPE; + } + plan->generic_node_count = plan->node_count - plan->fast_node_count; + if (plan->scratch_bytes != 0u) { + plan->scratch = (uint8_t*)malloc(plan->scratch_bytes); + if (plan->scratch == NULL) { + lw_set_error(error, LW_STATUS_OUT_OF_MEMORY, "unable to allocate fast scratch"); + lw_x64_rec_fast_plan_free(plan); + return LW_STATUS_OUT_OF_MEMORY; + } + } + lw_set_error(error, LW_STATUS_OK, ""); + *out_plan = plan; + return LW_STATUS_OK; +} + +lw_status lw_x64_rec_fast_plan_create(lw_session* session, + lw_x64_rec_fast_plan** out_plan, + lw_error* error) { + return lw_x64_rec_fast_plan_create_shared(session, NULL, out_plan, error); +} + +void lw_x64_rec_fast_plan_free(lw_x64_rec_fast_plan* plan) { + if (plan == NULL) return; + if (plan->nodes != NULL) { + uint32_t index; + for (index = 0u; index < plan->node_count; ++index) { + if (fast_physical_op_owns_conv(plan->nodes[index].kind)) { + fast_release_conv(&plan->nodes[index].data.conv); + } + if (fast_physical_op_owns_spatial(plan->nodes[index].kind)) { + fast_release_spatial(&plan->nodes[index].data.spatial); + } + } + } + if (plan->ops != NULL) { + uint32_t index; + for (index = 0u; index < plan->op_count; ++index) { + if (fast_physical_op_owns_conv(plan->ops[index].kind)) { + fast_release_conv(&plan->ops[index].data.conv); + } + if (fast_physical_op_owns_spatial(plan->ops[index].kind)) { + fast_release_spatial(&plan->ops[index].data.spatial); + } + } + } + free(plan->scratch); + free(plan->nhwc_workspace); + free(plan->tensors); + free(plan->consumer_count); + free(plan->elided_tensor); + free(plan->physical_lifetimes); + free(plan->semantic_to_physical); + free(plan->nodes); + free(plan->ops); + lw_layout_plan_free(&plan->layout); + free(plan); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_binary.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_binary.c new file mode 100644 index 00000000..8a1ae659 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_binary.c @@ -0,0 +1,266 @@ +#include "simd_kernels.h" + +#include +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_BINARY 1 +#else +# define LW_COMPILES_AVX2_BINARY 0 +#endif + +#if LW_COMPILES_AVX2_BINARY && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_binary_contiguous_f32( + lw_scalar_binary_op operation, + const float* left, + const float* right, + float* output, + uint64_t element_count) { + uint64_t index = 0u; +#if LW_COMPILES_AVX2_BINARY + if (operation == LW_SCALAR_BINARY_ADD) { + for (; index + 8u <= element_count; index += 8u) { + __m256 result = _mm256_add_ps(_mm256_loadu_ps(left + (size_t)index), + _mm256_loadu_ps(right + (size_t)index)); + _mm256_storeu_ps(output + (size_t)index, result); + } + } else if (operation == LW_SCALAR_BINARY_MUL) { + for (; index + 8u <= element_count; index += 8u) { + __m256 result = _mm256_mul_ps(_mm256_loadu_ps(left + (size_t)index), + _mm256_loadu_ps(right + (size_t)index)); + _mm256_storeu_ps(output + (size_t)index, result); + } + } else if (operation == LW_SCALAR_BINARY_DIV) { + for (; index + 8u <= element_count; index += 8u) { + __m256 result = _mm256_div_ps(_mm256_loadu_ps(left + (size_t)index), + _mm256_loadu_ps(right + (size_t)index)); + _mm256_storeu_ps(output + (size_t)index, result); + } + } +#endif + lw_scalar_binary_contiguous_f32(operation, left + (size_t)index, right + (size_t)index, + output + (size_t)index, element_count - index); +} + +#if LW_COMPILES_AVX2_BINARY && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_binary_right_scalar_f32( + lw_scalar_binary_op operation, + const float* left, + float right, + float* output, + uint64_t element_count) { + uint64_t index = 0u; +#if LW_COMPILES_AVX2_BINARY + __m256 right_values = _mm256_set1_ps(right); + if (operation == LW_SCALAR_BINARY_ADD) { + for (; index + 8u <= element_count; index += 8u) { + __m256 result = _mm256_add_ps(_mm256_loadu_ps(left + (size_t)index), right_values); + _mm256_storeu_ps(output + (size_t)index, result); + } + } else if (operation == LW_SCALAR_BINARY_MUL) { + for (; index + 8u <= element_count; index += 8u) { + __m256 result = _mm256_mul_ps(_mm256_loadu_ps(left + (size_t)index), right_values); + _mm256_storeu_ps(output + (size_t)index, result); + } + } else if (operation == LW_SCALAR_BINARY_DIV) { + for (; index + 8u <= element_count; index += 8u) { + __m256 result = _mm256_div_ps(_mm256_loadu_ps(left + (size_t)index), right_values); + _mm256_storeu_ps(output + (size_t)index, result); + } + } +#endif + lw_scalar_binary_right_scalar_f32(operation, left + (size_t)index, right, + output + (size_t)index, element_count - index); +} + +#if LW_COMPILES_AVX2_BINARY && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_relu_contiguous_f32(const float* input, float* output, uint64_t element_count) { + uint64_t index = 0u; +#if LW_COMPILES_AVX2_BINARY + __m256 zero = _mm256_setzero_ps(); + for (; index + 8u <= element_count; index += 8u) { + __m256 value = _mm256_max_ps(_mm256_loadu_ps(input + (size_t)index), zero); + _mm256_storeu_ps(output + (size_t)index, value); + } +#endif + for (; index < element_count; ++index) { + float value = input[(size_t)index]; + output[(size_t)index] = value > 0.0f ? value : 0.0f; + } +} + +#if LW_COMPILES_AVX2_BINARY && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_binary_channel_f32(lw_scalar_binary_op operation, const float* full, + const float* channel, float* output, uint64_t pixels, + uint32_t channels, int broadcast_is_left) { + uint64_t pixel; + if (full == NULL || channel == NULL || output == NULL || channels == 0u) return; + for (pixel = 0u; pixel < pixels; ++pixel) { + const float* source = full + (size_t)pixel * channels; + float* destination = output + (size_t)pixel * channels; + uint32_t c = 0u; +#if LW_COMPILES_AVX2_BINARY + for (; c + 8u <= channels; c += 8u) { + __m256 value = _mm256_loadu_ps(source + c); + __m256 scale = _mm256_loadu_ps(channel + c); + __m256 result; + if (operation == LW_SCALAR_BINARY_ADD) { + result = _mm256_add_ps(broadcast_is_left ? scale : value, + broadcast_is_left ? value : scale); + } else if (operation == LW_SCALAR_BINARY_SUB) { + result = _mm256_sub_ps(broadcast_is_left ? scale : value, + broadcast_is_left ? value : scale); + } else if (operation == LW_SCALAR_BINARY_MUL) { + result = _mm256_mul_ps(broadcast_is_left ? scale : value, + broadcast_is_left ? value : scale); + } else if (operation == LW_SCALAR_BINARY_DIV) { + result = _mm256_div_ps(broadcast_is_left ? scale : value, + broadcast_is_left ? value : scale); + } else { + break; + } + _mm256_storeu_ps(destination + c, result); + } +#endif + for (; c < channels; ++c) { + float left = broadcast_is_left ? channel[c] : source[c]; + float right = broadcast_is_left ? source[c] : channel[c]; + if (operation == LW_SCALAR_BINARY_ADD) destination[c] = left + right; + else if (operation == LW_SCALAR_BINARY_SUB) destination[c] = left - right; + else if (operation == LW_SCALAR_BINARY_MUL) destination[c] = left * right; + else if (operation == LW_SCALAR_BINARY_DIV) destination[c] = left / right; + else destination[c] = powf(left, right); + } + } +} + + +#if LW_COMPILES_AVX2_BINARY && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_binary_channel_nchw_f32(lw_scalar_binary_op operation, const float* full, + const float* channel, float* output, uint64_t spatial, + uint32_t channels, int broadcast_is_left) { + if (full == NULL || channel == NULL || output == NULL || channels == 0u) return; + for (uint32_t c = 0u; c < channels; ++c) { + const float* source = full + (size_t)c * spatial; + float* destination = output + (size_t)c * spatial; + uint64_t i = 0u; +#if LW_COMPILES_AVX2_BINARY + const __m256 scalar = _mm256_set1_ps(channel[c]); + for (; i + 8u <= spatial; i += 8u) { + const __m256 value = _mm256_loadu_ps(source + i); + __m256 result; + if (operation == LW_SCALAR_BINARY_ADD) { + result = _mm256_add_ps(broadcast_is_left ? scalar : value, + broadcast_is_left ? value : scalar); + } else if (operation == LW_SCALAR_BINARY_SUB) { + result = _mm256_sub_ps(broadcast_is_left ? scalar : value, + broadcast_is_left ? value : scalar); + } else if (operation == LW_SCALAR_BINARY_MUL) { + result = _mm256_mul_ps(broadcast_is_left ? scalar : value, + broadcast_is_left ? value : scalar); + } else if (operation == LW_SCALAR_BINARY_DIV) { + result = _mm256_div_ps(broadcast_is_left ? scalar : value, + broadcast_is_left ? value : scalar); + } else { + break; + } + _mm256_storeu_ps(destination + i, result); + } +#endif + for (; i < spatial; ++i) { + const float left = broadcast_is_left ? channel[c] : source[i]; + const float right = broadcast_is_left ? source[i] : channel[c]; + if (operation == LW_SCALAR_BINARY_ADD) destination[i] = left + right; + else if (operation == LW_SCALAR_BINARY_SUB) destination[i] = left - right; + else if (operation == LW_SCALAR_BINARY_MUL) destination[i] = left * right; + else if (operation == LW_SCALAR_BINARY_DIV) destination[i] = left / right; + else destination[i] = powf(left, right); + } + } +} + +#if LW_COMPILES_AVX2_BINARY && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,fma"))) +#endif +void lw_avx2_hard_sigmoid_contiguous_f32(const float* input, float* output, + uint64_t element_count, float alpha, float beta) { + uint64_t index = 0u; +#if LW_COMPILES_AVX2_BINARY + const __m256 alpha_v = _mm256_set1_ps(alpha); + const __m256 beta_v = _mm256_set1_ps(beta); + const __m256 zero = _mm256_setzero_ps(); + const __m256 one = _mm256_set1_ps(1.0f); + for (; index + 8u <= element_count; index += 8u) { + __m256 value = _mm256_fmadd_ps(_mm256_loadu_ps(input + (size_t)index), alpha_v, beta_v); + value = _mm256_max_ps(_mm256_min_ps(value, one), zero); + _mm256_storeu_ps(output + (size_t)index, value); + } +#endif + for (; index < element_count; ++index) { + float value = input[(size_t)index] * alpha + beta; + if (value < 0.0f) value = 0.0f; + else if (value > 1.0f) value = 1.0f; + output[(size_t)index] = value; + } +} + +#if LW_COMPILES_AVX2_BINARY && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_hard_sigmoid_exact_f32(const float* input, float* output, + uint64_t element_count, float alpha, float beta) { + uint64_t index = 0u; +#if LW_COMPILES_AVX2_BINARY + const __m256 alpha_v = _mm256_set1_ps(alpha); + const __m256 beta_v = _mm256_set1_ps(beta); + const __m256 zero = _mm256_setzero_ps(); + const __m256 one = _mm256_set1_ps(1.0f); + /* Separate multiply-add on purpose: the scalar reference evaluates + * alpha * x + beta with two roundings, and the FMA variant is not + * bit-identical. */ + for (; index + 8u <= element_count; index += 8u) { + __m256 value = + _mm256_add_ps(_mm256_mul_ps(_mm256_loadu_ps(input + (size_t)index), alpha_v), + beta_v); + value = _mm256_max_ps(_mm256_min_ps(value, one), zero); + _mm256_storeu_ps(output + (size_t)index, value); + } +#endif + for (; index < element_count; ++index) { + float value = input[(size_t)index] * alpha + beta; + if (value < 0.0f) value = 0.0f; + else if (value > 1.0f) value = 1.0f; + output[(size_t)index] = value; + } +} + +#if LW_COMPILES_AVX2_BINARY && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_relu_f32(const float* input, float* output, uint64_t element_count) { + uint64_t index = 0u; +#if LW_COMPILES_AVX2_BINARY + const __m256 zero = _mm256_setzero_ps(); + /* max(v, 0) matches the scalar v > 0 ? v : 0 for every value including + * NaN (max_ps returns the second operand when either is NaN). */ + for (; index + 8u <= element_count; index += 8u) { + _mm256_storeu_ps(output + (size_t)index, + _mm256_max_ps(_mm256_loadu_ps(input + (size_t)index), zero)); + } +#endif + for (; index < element_count; ++index) { + float value = input[(size_t)index]; + output[(size_t)index] = value > 0.0f ? value : 0.0f; + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_bitmap.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_bitmap.c new file mode 100644 index 00000000..a33bcff0 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_bitmap.c @@ -0,0 +1,206 @@ +#include "simd_kernels.h" + +/* AVX2 bitmap helpers for the DET postprocess: thresholding the probability + * map and computing the 4-neighborhood "interior" bitmap that replaces the + * per-pixel boundary probe during connected-component extraction. */ + +#include +#include +#include +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_BITMAP 1 +#else +# define LW_COMPILES_AVX2_BITMAP 0 +#endif + +#if LW_COMPILES_AVX2_BITMAP && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +int lw_avx2_threshold_bitmap_f32(const float* prediction, uint8_t* bitmap, + uint64_t pixel_count, float threshold) { +#if LW_COMPILES_AVX2_BITMAP + /* One 64-bit word of output per 8 input floats: each movemask bit expands + * to a full 0/1 byte through the 256-entry expansion table. */ + static const uint64_t mask_expand[256] = { + 0x0000000000000000ULL, 0x0000000000000001ULL, 0x0000000000000100ULL, + 0x0000000000000101ULL, 0x0000000000010000ULL, 0x0000000000010001ULL, + 0x0000000000010100ULL, 0x0000000000010101ULL, 0x0000000001000000ULL, + 0x0000000001000001ULL, 0x0000000001000100ULL, 0x0000000001000101ULL, + 0x0000000001010000ULL, 0x0000000001010001ULL, 0x0000000001010100ULL, + 0x0000000001010101ULL, 0x0000000100000000ULL, 0x0000000100000001ULL, + 0x0000000100000100ULL, 0x0000000100000101ULL, 0x0000000100010000ULL, + 0x0000000100010001ULL, 0x0000000100010100ULL, 0x0000000100010101ULL, + 0x0000000101000000ULL, 0x0000000101000001ULL, 0x0000000101000100ULL, + 0x0000000101000101ULL, 0x0000000101010000ULL, 0x0000000101010001ULL, + 0x0000000101010100ULL, 0x0000000101010101ULL, 0x0000010000000000ULL, + 0x0000010000000001ULL, 0x0000010000000100ULL, 0x0000010000000101ULL, + 0x0000010000010000ULL, 0x0000010000010001ULL, 0x0000010000010100ULL, + 0x0000010000010101ULL, 0x0000010001000000ULL, 0x0000010001000001ULL, + 0x0000010001000100ULL, 0x0000010001000101ULL, 0x0000010001010000ULL, + 0x0000010001010001ULL, 0x0000010001010100ULL, 0x0000010001010101ULL, + 0x0000010100000000ULL, 0x0000010100000001ULL, 0x0000010100000100ULL, + 0x0000010100000101ULL, 0x0000010100010000ULL, 0x0000010100010001ULL, + 0x0000010100010100ULL, 0x0000010100010101ULL, 0x0000010101000000ULL, + 0x0000010101000001ULL, 0x0000010101000100ULL, 0x0000010101000101ULL, + 0x0000010101010000ULL, 0x0000010101010001ULL, 0x0000010101010100ULL, + 0x0000010101010101ULL, 0x0001000000000000ULL, 0x0001000000000001ULL, + 0x0001000000000100ULL, 0x0001000000000101ULL, 0x0001000000010000ULL, + 0x0001000000010001ULL, 0x0001000000010100ULL, 0x0001000000010101ULL, + 0x0001000001000000ULL, 0x0001000001000001ULL, 0x0001000001000100ULL, + 0x0001000001000101ULL, 0x0001000001010000ULL, 0x0001000001010001ULL, + 0x0001000001010100ULL, 0x0001000001010101ULL, 0x0001000100000000ULL, + 0x0001000100000001ULL, 0x0001000100000100ULL, 0x0001000100000101ULL, + 0x0001000100010000ULL, 0x0001000100010001ULL, 0x0001000100010100ULL, + 0x0001000100010101ULL, 0x0001000101000000ULL, 0x0001000101000001ULL, + 0x0001000101000100ULL, 0x0001000101000101ULL, 0x0001000101010000ULL, + 0x0001000101010001ULL, 0x0001000101010100ULL, 0x0001000101010101ULL, + 0x0001010000000000ULL, 0x0001010000000001ULL, 0x0001010000000100ULL, + 0x0001010000000101ULL, 0x0001010000010000ULL, 0x0001010000010001ULL, + 0x0001010000010100ULL, 0x0001010000010101ULL, 0x0001010001000000ULL, + 0x0001010001000001ULL, 0x0001010001000100ULL, 0x0001010001000101ULL, + 0x0001010001010000ULL, 0x0001010001010001ULL, 0x0001010001010100ULL, + 0x0001010001010101ULL, 0x0001010100000000ULL, 0x0001010100000001ULL, + 0x0001010100000100ULL, 0x0001010100000101ULL, 0x0001010100010000ULL, + 0x0001010100010001ULL, 0x0001010100010100ULL, 0x0001010100010101ULL, + 0x0001010101000000ULL, 0x0001010101000001ULL, 0x0001010101000100ULL, + 0x0001010101000101ULL, 0x0001010101010000ULL, 0x0001010101010001ULL, + 0x0001010101010100ULL, 0x0001010101010101ULL, 0x0100000000000000ULL, + 0x0100000000000001ULL, 0x0100000000000100ULL, 0x0100000000000101ULL, + 0x0100000000010000ULL, 0x0100000000010001ULL, 0x0100000000010100ULL, + 0x0100000000010101ULL, 0x0100000001000000ULL, 0x0100000001000001ULL, + 0x0100000001000100ULL, 0x0100000001000101ULL, 0x0100000001010000ULL, + 0x0100000001010001ULL, 0x0100000001010100ULL, 0x0100000001010101ULL, + 0x0100000100000000ULL, 0x0100000100000001ULL, 0x0100000100000100ULL, + 0x0100000100000101ULL, 0x0100000100010000ULL, 0x0100000100010001ULL, + 0x0100000100010100ULL, 0x0100000100010101ULL, 0x0100000101000000ULL, + 0x0100000101000001ULL, 0x0100000101000100ULL, 0x0100000101000101ULL, + 0x0100000101010000ULL, 0x0100000101010001ULL, 0x0100000101010100ULL, + 0x0100000101010101ULL, 0x0100010000000000ULL, 0x0100010000000001ULL, + 0x0100010000000100ULL, 0x0100010000000101ULL, 0x0100010000010000ULL, + 0x0100010000010001ULL, 0x0100010000010100ULL, 0x0100010000010101ULL, + 0x0100010001000000ULL, 0x0100010001000001ULL, 0x0100010001000100ULL, + 0x0100010001000101ULL, 0x0100010001010000ULL, 0x0100010001010001ULL, + 0x0100010001010100ULL, 0x0100010001010101ULL, 0x0100010100000000ULL, + 0x0100010100000001ULL, 0x0100010100000100ULL, 0x0100010100000101ULL, + 0x0100010100010000ULL, 0x0100010100010001ULL, 0x0100010100010100ULL, + 0x0100010100010101ULL, 0x0100010101000000ULL, 0x0100010101000001ULL, + 0x0100010101000100ULL, 0x0100010101000101ULL, 0x0100010101010000ULL, + 0x0100010101010001ULL, 0x0100010101010100ULL, 0x0100010101010101ULL, + 0x0101000000000000ULL, 0x0101000000000001ULL, 0x0101000000000100ULL, + 0x0101000000000101ULL, 0x0101000000010000ULL, 0x0101000000010001ULL, + 0x0101000000010100ULL, 0x0101000000010101ULL, 0x0101000001000000ULL, + 0x0101000001000001ULL, 0x0101000001000100ULL, 0x0101000001000101ULL, + 0x0101000001010000ULL, 0x0101000001010001ULL, 0x0101000001010100ULL, + 0x0101000001010101ULL, 0x0101000100000000ULL, 0x0101000100000001ULL, + 0x0101000100000100ULL, 0x0101000100000101ULL, 0x0101000100010000ULL, + 0x0101000100010001ULL, 0x0101000100010100ULL, 0x0101000100010101ULL, + 0x0101000101000000ULL, 0x0101000101000001ULL, 0x0101000101000100ULL, + 0x0101000101000101ULL, 0x0101000101010000ULL, 0x0101000101010001ULL, + 0x0101000101010100ULL, 0x0101000101010101ULL, 0x0101010000000000ULL, + 0x0101010000000001ULL, 0x0101010000000100ULL, 0x0101010000000101ULL, + 0x0101010000010000ULL, 0x0101010000010001ULL, 0x0101010000010100ULL, + 0x0101010000010101ULL, 0x0101010001000000ULL, 0x0101010001000001ULL, + 0x0101010001000100ULL, 0x0101010001000101ULL, 0x0101010001010000ULL, + 0x0101010001010001ULL, 0x0101010001010100ULL, 0x0101010001010101ULL, + 0x0101010100000000ULL, 0x0101010100000001ULL, 0x0101010100000100ULL, + 0x0101010100000101ULL, 0x0101010100010000ULL, 0x0101010100010001ULL, + 0x0101010100010100ULL, 0x0101010100010101ULL, 0x0101010101000000ULL, + 0x0101010101000001ULL, 0x0101010101000100ULL, 0x0101010101000101ULL, + 0x0101010101010000ULL, 0x0101010101010001ULL, 0x0101010101010100ULL, + 0x0101010101010101ULL}; + const __m256 v_threshold = _mm256_set1_ps(threshold); + const __m256 v_abs_mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff)); + const __m256 v_infinity = _mm256_set1_ps(INFINITY); + uint64_t index; + for (index = 0u; index + 8u <= pixel_count; index += 8u) { + __m256 values = _mm256_loadu_ps(prediction + (size_t)index); + __m256 comparison = + _mm256_cmp_ps(_mm256_and_ps(values, v_abs_mask), v_infinity, _CMP_NLT_UQ); + if (_mm256_movemask_ps(comparison) != 0) { + return -1; /* non-finite value (NaN or infinite) */ + } + { + int mask = _mm256_movemask_ps( + _mm256_cmp_ps(values, v_threshold, _CMP_GT_OQ)); + uint64_t word = mask_expand[(uint32_t)mask]; + memcpy(bitmap + (size_t)index, &word, sizeof(word)); + } + } + for (; index < pixel_count; ++index) { + float value = prediction[(size_t)index]; + if (!isfinite(value)) { + return -1; + } + bitmap[(size_t)index] = value > threshold ? 1u : 0u; + } + return 0; +#else + uint64_t index; + (void)threshold; + for (index = 0u; index < pixel_count; ++index) { + float value = prediction[(size_t)index]; + if (!isfinite(value)) { + return -1; + } + bitmap[(size_t)index] = value > threshold ? 1u : 0u; + } + return 0; +#endif +} + +#if LW_COMPILES_AVX2_BITMAP && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_interior_bitmap_u8(const uint8_t* bitmap, uint8_t* interior, + uint32_t width, uint32_t height) { + /* 4-neighborhood interior: interior[p] is non-zero only when all four + * orthogonal neighbors are foreground. Border pixels stay zero, which + * reproduces the per-pixel boundary probe exactly. */ + uint32_t y; + memset(interior, 0, (size_t)width * height); + if (width < 3u || height < 3u) return; +#if LW_COMPILES_AVX2_BITMAP + for (y = 1u; y + 1u < height; ++y) { + const uint8_t* previous = bitmap + (size_t)(y - 1u) * width; + const uint8_t* current = bitmap + (size_t)y * width; + const uint8_t* next = bitmap + (size_t)(y + 1u) * width; + uint8_t* destination = interior + (size_t)y * width; + uint32_t x = 1u; + for (; x + 33u <= width; x += 32u) { + __m256i vertical = _mm256_and_si256( + _mm256_loadu_si256((const __m256i*)(const void*)(previous + x)), + _mm256_loadu_si256((const __m256i*)(const void*)(next + x))); + __m256i horizontal = _mm256_and_si256( + _mm256_loadu_si256((const __m256i*)(const void*)(current + x - 1u)), + _mm256_loadu_si256((const __m256i*)(const void*)(current + x + 1u))); + _mm256_storeu_si256((__m256i*)(void*)(destination + x), + _mm256_and_si256(vertical, horizontal)); + } + for (; x + 1u < width; ++x) { + if (previous[x - 1u] != 0u && previous[x + 1u] != 0u && + current[x - 1u] != 0u && current[x + 1u] != 0u && + next[x - 1u] != 0u && next[x + 1u] != 0u) { + destination[x] = 1u; + } + } + } +#else + for (y = 1u; y + 1u < height; ++y) { + const uint8_t* previous = bitmap + (size_t)(y - 1u) * width; + const uint8_t* current = bitmap + (size_t)y * width; + const uint8_t* next = bitmap + (size_t)(y + 1u) * width; + uint8_t* destination = interior + (size_t)y * width; + uint32_t x; + for (x = 1u; x + 1u < width; ++x) { + if (previous[x - 1u] != 0u && previous[x + 1u] != 0u && + current[x - 1u] != 0u && current[x + 1u] != 0u && + next[x - 1u] != 0u && next[x + 1u] != 0u) { + destination[x] = 1u; + } + } + } +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv1x1.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv1x1.c new file mode 100644 index 00000000..24d1362f --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv1x1.c @@ -0,0 +1,90 @@ +#include "simd_kernels.h" + +#include "scalar_kernels.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_CONV1X1 1 +#else +# define LW_COMPILES_AVX2_CONV1X1 0 +#endif + +#if LW_COMPILES_AVX2_CONV1X1 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv1x1_unit_f32( + const float* input, + const float* weights, + const float* bias, + float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4], + uint32_t groups, + uint32_t input_channels_per_group, + uint32_t output_channels_per_group) { +#if LW_COMPILES_AVX2_CONV1X1 + uint64_t channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + uint32_t group; + for (group = 0u; group < groups; ++group) { + uint32_t input_channel_start = group * input_channels_per_group; + uint32_t output_channel_start = group * output_channels_per_group; + const float* group_input = + input + + (size_t)(((uint64_t)batch * (uint32_t)input_dimensions[1] + input_channel_start) * + channel_plane); + float* group_output = + output + + (size_t)(((uint64_t)batch * (uint32_t)output_dimensions[1] + output_channel_start) * + channel_plane); + uint32_t output_channel_in_group; + for (output_channel_in_group = 0u; output_channel_in_group < output_channels_per_group; + ++output_channel_in_group) { + uint32_t output_channel = output_channel_start + output_channel_in_group; + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * input_channels_per_group); + float* output_channel_data = + group_output + (size_t)((uint64_t)output_channel_in_group * channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel_in_group; + for (; spatial + 8u <= channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel_in_group = 0u; input_channel_in_group < input_channels_per_group; + ++input_channel_in_group) { + const float* input_channel_data = + group_input + (size_t)((uint64_t)input_channel_in_group * channel_plane); + __m256 weight_values = + _mm256_set1_ps(output_channel_weights[input_channel_in_group]); + spatial = 0u; + for (; spatial + 8u <= channel_plane; spatial += 8u) { + __m256 output_values = + _mm256_loadu_ps(output_channel_data + (size_t)spatial); + __m256 input_values = _mm256_loadu_ps(input_channel_data + (size_t)spatial); + output_values = _mm256_add_ps(output_values, + _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_channel_data + (size_t)spatial, output_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] += + input_channel_data[(size_t)spatial] * + output_channel_weights[input_channel_in_group]; + } + } + } + } + } +#else + lw_scalar_conv1x1_unit_f32(input, weights, bias, output, input_dimensions, output_dimensions, + groups, input_channels_per_group, output_channels_per_group); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv2x2.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv2x2.c new file mode 100644 index 00000000..15b9969a --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv2x2.c @@ -0,0 +1,205 @@ +#include "simd_kernels.h" + +#include "scalar_kernels.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_CONV2X2 1 +#else +# define LW_COMPILES_AVX2_CONV2X2 0 +#endif + +#if LW_COMPILES_AVX2_CONV2X2 +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static void conv2x2_unit_pad_end1_four_outputs_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { + uint32_t height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + output_channel += 4u) { + const uint64_t weights_per_output = (uint64_t)(uint32_t)input_dimensions[1] * 4u; + const float* weights0 = weights + (size_t)((uint64_t)(output_channel + 0u) * weights_per_output); + const float* weights1 = weights + (size_t)((uint64_t)(output_channel + 1u) * weights_per_output); + const float* weights2 = weights + (size_t)((uint64_t)(output_channel + 2u) * weights_per_output); + const float* weights3 = weights + (size_t)((uint64_t)(output_channel + 3u) * weights_per_output); + float* output0 = batch_output + (size_t)((uint64_t)(output_channel + 0u) * channel_plane); + float* output1 = batch_output + (size_t)((uint64_t)(output_channel + 1u) * channel_plane); + float* output2 = batch_output + (size_t)((uint64_t)(output_channel + 2u) * channel_plane); + float* output3 = batch_output + (size_t)((uint64_t)(output_channel + 3u) * channel_plane); + const float initial0 = bias == NULL ? 0.0f : bias[output_channel + 0u]; + const float initial1 = bias == NULL ? 0.0f : bias[output_channel + 1u]; + const float initial2 = bias == NULL ? 0.0f : bias[output_channel + 2u]; + const float initial3 = bias == NULL ? 0.0f : bias[output_channel + 3u]; + __m256 initial_values0 = _mm256_set1_ps(initial0); + __m256 initial_values1 = _mm256_set1_ps(initial1); + __m256 initial_values2 = _mm256_set1_ps(initial2); + __m256 initial_values3 = _mm256_set1_ps(initial3); + uint64_t spatial = 0u; + for (; spatial + 8u <= channel_plane; spatial += 8u) { + _mm256_storeu_ps(output0 + (size_t)spatial, initial_values0); + _mm256_storeu_ps(output1 + (size_t)spatial, initial_values1); + _mm256_storeu_ps(output2 + (size_t)spatial, initial_values2); + _mm256_storeu_ps(output3 + (size_t)spatial, initial_values3); + } + for (; spatial < channel_plane; ++spatial) { + output0[(size_t)spatial] = initial0; + output1[(size_t)spatial] = initial1; + output2[(size_t)spatial] = initial2; + output3[(size_t)spatial] = initial3; + } + uint32_t input_channel; + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weights0_channel = weights0 + (size_t)input_channel * 4u; + const float* weights1_channel = weights1 + (size_t)input_channel * 4u; + const float* weights2_channel = weights2 + (size_t)input_channel * 4u; + const float* weights3_channel = weights3 + (size_t)input_channel * 4u; + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 2u; ++kernel_y) { + uint32_t output_y; + uint32_t output_y_end = height - kernel_y; + for (output_y = 0u; output_y < output_y_end; ++output_y) { + const float* input_row = input_channel_data + + (size_t)((uint64_t)(output_y + kernel_y) * width); + float* output_row0 = output0 + (size_t)((uint64_t)output_y * width); + float* output_row1 = output1 + (size_t)((uint64_t)output_y * width); + float* output_row2 = output2 + (size_t)((uint64_t)output_y * width); + float* output_row3 = output3 + (size_t)((uint64_t)output_y * width); + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 2u; ++kernel_x) { + uint32_t output_x = 0u; + uint32_t output_x_end = width - kernel_x; + __m256 weight0 = _mm256_set1_ps(weights0_channel[kernel_y * 2u + kernel_x]); + __m256 weight1 = _mm256_set1_ps(weights1_channel[kernel_y * 2u + kernel_x]); + __m256 weight2 = _mm256_set1_ps(weights2_channel[kernel_y * 2u + kernel_x]); + __m256 weight3 = _mm256_set1_ps(weights3_channel[kernel_y * 2u + kernel_x]); + const float* input_values = input_row + kernel_x; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + __m256 values = _mm256_loadu_ps(input_values + output_x); + __m256 values0 = _mm256_loadu_ps(output_row0 + output_x); + __m256 values1 = _mm256_loadu_ps(output_row1 + output_x); + __m256 values2 = _mm256_loadu_ps(output_row2 + output_x); + __m256 values3 = _mm256_loadu_ps(output_row3 + output_x); + _mm256_storeu_ps(output_row0 + output_x, + _mm256_add_ps(values0, _mm256_mul_ps(values, weight0))); + _mm256_storeu_ps(output_row1 + output_x, + _mm256_add_ps(values1, _mm256_mul_ps(values, weight1))); + _mm256_storeu_ps(output_row2 + output_x, + _mm256_add_ps(values2, _mm256_mul_ps(values, weight2))); + _mm256_storeu_ps(output_row3 + output_x, + _mm256_add_ps(values3, _mm256_mul_ps(values, weight3))); + } + for (; output_x < output_x_end; ++output_x) { + float value = input_values[output_x]; + output_row0[output_x] += value * weights0_channel[kernel_y * 2u + kernel_x]; + output_row1[output_x] += value * weights1_channel[kernel_y * 2u + kernel_x]; + output_row2[output_x] += value * weights2_channel[kernel_y * 2u + kernel_x]; + output_row3[output_x] += value * weights3_channel[kernel_y * 2u + kernel_x]; + } + } + } + } + } + } + } +} +#endif + +#if LW_COMPILES_AVX2_CONV2X2 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv2x2_unit_pad_end1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV2X2 + if ((uint32_t)output_dimensions[1] % 4u == 0u) { + conv2x2_unit_pad_end1_four_outputs_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + return; + } + uint32_t height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * (uint32_t)input_dimensions[1] * 4u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel; + for (; spatial + 8u <= channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 4u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 2u; ++kernel_y) { + uint32_t output_y_end = height - kernel_y; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 2u; ++kernel_x) { + uint32_t output_x_end = width - kernel_x; + __m256 weight_values = + _mm256_set1_ps(weight_channel_data[kernel_y * 2u + kernel_x]); + uint32_t output_y; + for (output_y = 0u; output_y < output_y_end; ++output_y) { + const float* input_row = + input_channel_data + + (size_t)((uint64_t)(output_y + kernel_y) * width + kernel_x); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x = 0u; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + __m256 input_values = _mm256_loadu_ps(input_row + output_x); + __m256 output_values = _mm256_loadu_ps(output_row + output_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + output_row[output_x] += + input_row[output_x] * + weight_channel_data[kernel_y * 2u + kernel_x]; + } + } + } + } + } + } + } +#else + lw_scalar_conv2x2_unit_pad_end1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv3x3.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv3x3.c new file mode 100644 index 00000000..05eb193a --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv3x3.c @@ -0,0 +1,1141 @@ +#include "simd_kernels.h" + +#include "packed_conv3x3_internal.h" +#include "scalar_kernels.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_CONV3X3 1 +#else +# define LW_COMPILES_AVX2_CONV3X3 0 +#endif + +#if LW_COMPILES_AVX2_CONV3X3 +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static void +conv3x3_unit_pad1_four_outputs_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t height = (uint32_t)input_dimensions[2]; + const uint32_t width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint64_t channel_plane = (uint64_t)height * width; + const uint64_t weights_per_output = (uint64_t)input_channels * 9u; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < output_channels; output_channel += 4u) { + const float* weights_0 = + weights + (size_t)((uint64_t)(output_channel + 0u) * weights_per_output); + const float* weights_1 = + weights + (size_t)((uint64_t)(output_channel + 1u) * weights_per_output); + const float* weights_2 = + weights + (size_t)((uint64_t)(output_channel + 2u) * weights_per_output); + const float* weights_3 = + weights + (size_t)((uint64_t)(output_channel + 3u) * weights_per_output); + float* output_0 = + batch_output + (size_t)((uint64_t)(output_channel + 0u) * channel_plane); + float* output_1 = + batch_output + (size_t)((uint64_t)(output_channel + 1u) * channel_plane); + float* output_2 = + batch_output + (size_t)((uint64_t)(output_channel + 2u) * channel_plane); + float* output_3 = + batch_output + (size_t)((uint64_t)(output_channel + 3u) * channel_plane); + const float initial_0 = bias == NULL ? 0.0f : bias[output_channel + 0u]; + const float initial_1 = bias == NULL ? 0.0f : bias[output_channel + 1u]; + const float initial_2 = bias == NULL ? 0.0f : bias[output_channel + 2u]; + const float initial_3 = bias == NULL ? 0.0f : bias[output_channel + 3u]; + uint32_t output_y; + for (output_y = 0u; output_y < height; ++output_y) { + const size_t output_row_offset = (size_t)((uint64_t)output_y * width); + uint32_t output_x = 0u; + while (output_x < width) { + const int full_height = output_y != 0u && output_y + 1u < height; + const int full_width = output_x != 0u && output_x + 8u < width; + if (full_height && full_width) { + __m256 accumulators_0 = _mm256_set1_ps(initial_0); + __m256 accumulators_1 = _mm256_set1_ps(initial_1); + __m256 accumulators_2 = _mm256_set1_ps(initial_2); + __m256 accumulators_3 = _mm256_set1_ps(initial_3); + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const size_t weight_offset = (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const uint32_t input_y = output_y + kernel_y - 1u; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const uint32_t input_x = output_x + kernel_x - 1u; + const size_t weight_index = + weight_offset + (size_t)kernel_y * 3u + kernel_x; + const __m256 input_values = + _mm256_loadu_ps(input_row + input_x); + accumulators_0 = _mm256_add_ps( + accumulators_0, + _mm256_mul_ps(input_values, + _mm256_set1_ps(weights_0[weight_index]))); + accumulators_1 = _mm256_add_ps( + accumulators_1, + _mm256_mul_ps(input_values, + _mm256_set1_ps(weights_1[weight_index]))); + accumulators_2 = _mm256_add_ps( + accumulators_2, + _mm256_mul_ps(input_values, + _mm256_set1_ps(weights_2[weight_index]))); + accumulators_3 = _mm256_add_ps( + accumulators_3, + _mm256_mul_ps(input_values, + _mm256_set1_ps(weights_3[weight_index]))); + } + } + } + _mm256_storeu_ps(output_0 + output_row_offset + output_x, accumulators_0); + _mm256_storeu_ps(output_1 + output_row_offset + output_x, accumulators_1); + _mm256_storeu_ps(output_2 + output_row_offset + output_x, accumulators_2); + _mm256_storeu_ps(output_3 + output_row_offset + output_x, accumulators_3); + output_x += 8u; + } else { + float accumulator_0 = initial_0; + float accumulator_1 = initial_1; + float accumulator_2 = initial_2; + float accumulator_3 = initial_3; + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const size_t weight_offset = (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const int64_t input_y = (int64_t)output_y + kernel_y - 1; + uint32_t kernel_x; + if (input_y < 0 || input_y >= (int64_t)height) { + continue; + } + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const int64_t input_x = (int64_t)output_x + kernel_x - 1; + if (input_x >= 0 && input_x < (int64_t)width) { + const size_t input_index = + (size_t)((uint64_t)(uint32_t)input_y * width + + (uint32_t)input_x); + const size_t weight_index = + weight_offset + (size_t)kernel_y * 3u + kernel_x; + const float value = input_channel_data[input_index]; + accumulator_0 += value * weights_0[weight_index]; + accumulator_1 += value * weights_1[weight_index]; + accumulator_2 += value * weights_2[weight_index]; + accumulator_3 += value * weights_3[weight_index]; + } + } + } + } + output_0[output_row_offset + output_x] = accumulator_0; + output_1[output_row_offset + output_x] = accumulator_1; + output_2[output_row_offset + output_x] = accumulator_2; + output_3[output_row_offset + output_x] = accumulator_3; + ++output_x; + } + } + } + } + } +} +#endif + +#if LW_COMPILES_AVX2_CONV3X3 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv3x3_unit_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV3X3 + if ((uint32_t)output_dimensions[1] % 4u == 0u) { + conv3x3_unit_pad1_four_outputs_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + return; + } + uint32_t height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * (uint32_t)input_dimensions[1] * 9u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel; + for (; spatial + 8u <= channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = kernel_y == 2u ? height - 1u : height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = kernel_x == 2u ? width - 1u : width; + float weight = weight_channel_data[kernel_y * 3u + kernel_x]; + __m256 weight_values = _mm256_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y + kernel_y - 1u; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + uint32_t input_x = output_x + kernel_x - 1u; + __m256 input_values = _mm256_loadu_ps(input_row + input_x); + __m256 output_values = _mm256_loadu_ps(output_row + output_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 1u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } + } +#else + lw_scalar_conv3x3_unit_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); +#endif +} + +#if LW_COMPILES_AVX2_CONV3X3 +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static void +conv3x3_stride2_pad1_output_stream_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + uint32_t input_height = (uint32_t)input_dimensions[2]; + uint32_t input_width = (uint32_t)input_dimensions[3]; + uint32_t output_height = (uint32_t)output_dimensions[2]; + uint32_t output_width = (uint32_t)output_dimensions[3]; + uint64_t input_channel_plane = (uint64_t)input_height * input_width; + uint64_t output_channel_plane = (uint64_t)output_height * output_width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * input_channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * + output_channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * (uint32_t)input_dimensions[1] * 9u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * output_channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel; + for (; spatial + 8u <= output_channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < output_channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * input_channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = output_height; + uint32_t kernel_x; + if (kernel_y == 2u && (uint64_t)(output_y_end - 1u) * 2u + 1u >= input_height) { + --output_y_end; + } + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = output_width; + float weight = weight_channel_data[(size_t)kernel_y * 3u + kernel_x]; + __m256 weight_values = _mm256_set1_ps(weight); + uint32_t output_y; + if (kernel_x == 2u && + (uint64_t)(output_x_end - 1u) * 2u + 1u >= input_width) { + --output_x_end; + } + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y * 2u - 1u + kernel_y; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * input_width); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * output_width); + uint32_t output_x = output_x_begin; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + uint32_t input_x = output_x * 2u - 1u + kernel_x; + if ((uint64_t)input_x + 15u >= input_width) { + break; + } + { + __m256 first = _mm256_loadu_ps(input_row + input_x); + __m256 second = _mm256_loadu_ps(input_row + input_x + 8u); + __m128 first_even = _mm_shuffle_ps( + _mm256_castps256_ps128(first), + _mm256_extractf128_ps(first, 1), _MM_SHUFFLE(2, 0, 2, 0)); + __m128 second_even = _mm_shuffle_ps( + _mm256_castps256_ps128(second), + _mm256_extractf128_ps(second, 1), _MM_SHUFFLE(2, 0, 2, 0)); + __m256 input_values = _mm256_castps128_ps256(first_even); + __m256 output_values; + input_values = + _mm256_insertf128_ps(input_values, second_even, 1); + output_values = _mm256_loadu_ps(output_row + output_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + } + for (; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x * 2u - 1u + kernel_x; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } + } +} + +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static void +conv3x3_stride2_pad1_eight_outputs_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t input_height = (uint32_t)input_dimensions[2]; + const uint32_t input_width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint32_t output_height = (uint32_t)output_dimensions[2]; + const uint32_t output_width = (uint32_t)output_dimensions[3]; + const uint64_t input_channel_plane = (uint64_t)input_height * input_width; + const uint64_t output_channel_plane = (uint64_t)output_height * output_width; + const uint64_t weights_per_output = (uint64_t)input_channels * 9u; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * input_channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * output_channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < output_channels; output_channel += 8u) { + const float* output_weights[8]; + float* output_planes[8]; + float initial[8]; + uint32_t lane; + for (lane = 0u; lane < 8u; ++lane) { + output_weights[lane] = + weights + (size_t)((uint64_t)(output_channel + lane) * weights_per_output); + output_planes[lane] = + batch_output + (size_t)((uint64_t)(output_channel + lane) * output_channel_plane); + initial[lane] = bias == NULL ? 0.0f : bias[output_channel + lane]; + } + uint32_t output_y; + for (output_y = 0u; output_y < output_height; ++output_y) { + const size_t output_row_offset = (size_t)((uint64_t)output_y * output_width); + uint32_t output_x = 0u; + while (output_x < output_width) { + const int full_height = output_y != 0u && (uint64_t)output_y * 2u + 1u < input_height; + const int full_width = output_x != 0u && output_x + 8u <= output_width && + ((uint64_t)output_x + 7u) * 2u + 1u < input_width; + if (full_height && full_width) { + __m256 accumulators[8]; + uint32_t input_channel; + for (lane = 0u; lane < 8u; ++lane) { + accumulators[lane] = _mm256_set1_ps(initial[lane]); + } + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * input_channel_plane); + const size_t weight_offset = (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const uint32_t input_y = output_y * 2u - 1u + kernel_y; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * input_width); + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const uint32_t input_x = output_x * 2u - 1u + kernel_x; + const size_t weight_index = + weight_offset + (size_t)kernel_y * 3u + kernel_x; + __m256 first = _mm256_loadu_ps(input_row + input_x); + __m256 second = _mm256_loadu_ps(input_row + input_x + 8u); + __m128 first_even = _mm_shuffle_ps( + _mm256_castps256_ps128(first), + _mm256_extractf128_ps(first, 1), _MM_SHUFFLE(2, 0, 2, 0)); + __m128 second_even = _mm_shuffle_ps( + _mm256_castps256_ps128(second), + _mm256_extractf128_ps(second, 1), _MM_SHUFFLE(2, 0, 2, 0)); + __m256 input_values = _mm256_castps128_ps256(first_even); + input_values = _mm256_insertf128_ps(input_values, second_even, 1); + for (lane = 0u; lane < 8u; ++lane) { + accumulators[lane] = _mm256_add_ps( + accumulators[lane], + _mm256_mul_ps(input_values, + _mm256_set1_ps(output_weights[lane][weight_index]))); + } + } + } + } + for (lane = 0u; lane < 8u; ++lane) { + _mm256_storeu_ps(output_planes[lane] + output_row_offset + output_x, + accumulators[lane]); + } + output_x += 8u; + } else { + float accumulators[8]; + uint32_t input_channel; + for (lane = 0u; lane < 8u; ++lane) { + accumulators[lane] = initial[lane]; + } + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * input_channel_plane); + const size_t weight_offset = (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const int64_t input_y = (int64_t)output_y * 2 - 1 + kernel_y; + uint32_t kernel_x; + if (input_y < 0 || input_y >= (int64_t)input_height) { + continue; + } + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const int64_t input_x = (int64_t)output_x * 2 - 1 + kernel_x; + if (input_x >= 0 && input_x < (int64_t)input_width) { + const size_t input_index = + (size_t)((uint64_t)(uint32_t)input_y * input_width + + (uint32_t)input_x); + const size_t weight_index = + weight_offset + (size_t)kernel_y * 3u + kernel_x; + const float value = input_channel_data[input_index]; + for (lane = 0u; lane < 8u; ++lane) { + accumulators[lane] += value * output_weights[lane][weight_index]; + } + } + } + } + } + for (lane = 0u; lane < 8u; ++lane) { + output_planes[lane][output_row_offset + output_x] = accumulators[lane]; + } + ++output_x; + } + } + } + } + } +} + +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static void +conv3x3_stride2_pad1_four_outputs_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t input_height = (uint32_t)input_dimensions[2]; + const uint32_t input_width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint32_t output_height = (uint32_t)output_dimensions[2]; + const uint32_t output_width = (uint32_t)output_dimensions[3]; + const uint64_t input_channel_plane = (uint64_t)input_height * input_width; + const uint64_t output_channel_plane = (uint64_t)output_height * output_width; + const uint64_t weights_per_output = (uint64_t)input_channels * 9u; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * input_channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * output_channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < output_channels; output_channel += 4u) { + const float* weights_0 = + weights + (size_t)((uint64_t)(output_channel + 0u) * weights_per_output); + const float* weights_1 = + weights + (size_t)((uint64_t)(output_channel + 1u) * weights_per_output); + const float* weights_2 = + weights + (size_t)((uint64_t)(output_channel + 2u) * weights_per_output); + const float* weights_3 = + weights + (size_t)((uint64_t)(output_channel + 3u) * weights_per_output); + float* output_0 = + batch_output + (size_t)((uint64_t)(output_channel + 0u) * output_channel_plane); + float* output_1 = + batch_output + (size_t)((uint64_t)(output_channel + 1u) * output_channel_plane); + float* output_2 = + batch_output + (size_t)((uint64_t)(output_channel + 2u) * output_channel_plane); + float* output_3 = + batch_output + (size_t)((uint64_t)(output_channel + 3u) * output_channel_plane); + const float initial_0 = bias == NULL ? 0.0f : bias[output_channel + 0u]; + const float initial_1 = bias == NULL ? 0.0f : bias[output_channel + 1u]; + const float initial_2 = bias == NULL ? 0.0f : bias[output_channel + 2u]; + const float initial_3 = bias == NULL ? 0.0f : bias[output_channel + 3u]; + uint32_t output_y; + for (output_y = 0u; output_y < output_height; ++output_y) { + const size_t output_row_offset = (size_t)((uint64_t)output_y * output_width); + uint32_t output_x = 0u; + while (output_x < output_width) { + const int full_height = + output_y != 0u && (uint64_t)output_y * 2u + 1u < input_height; + const int full_width = output_x != 0u && output_x + 8u <= output_width && + ((uint64_t)output_x + 7u) * 2u + 1u < input_width; + if (full_height && full_width) { + __m256 accumulators_0 = _mm256_set1_ps(initial_0); + __m256 accumulators_1 = _mm256_set1_ps(initial_1); + __m256 accumulators_2 = _mm256_set1_ps(initial_2); + __m256 accumulators_3 = _mm256_set1_ps(initial_3); + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * input_channel_plane); + const size_t weight_offset = (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const uint32_t input_y = output_y * 2u - 1u + kernel_y; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * input_width); + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const uint32_t input_x = output_x * 2u - 1u + kernel_x; + __m256 first = _mm256_loadu_ps(input_row + input_x); + __m256 second = _mm256_loadu_ps(input_row + input_x + 8u); + __m128 first_even = _mm_shuffle_ps( + _mm256_castps256_ps128(first), + _mm256_extractf128_ps(first, 1), _MM_SHUFFLE(2, 0, 2, 0)); + __m128 second_even = _mm_shuffle_ps( + _mm256_castps256_ps128(second), + _mm256_extractf128_ps(second, 1), _MM_SHUFFLE(2, 0, 2, 0)); + __m256 input_values = _mm256_castps128_ps256(first_even); + const size_t weight_index = + weight_offset + (size_t)kernel_y * 3u + kernel_x; + input_values = + _mm256_insertf128_ps(input_values, second_even, 1); + accumulators_0 = _mm256_add_ps( + accumulators_0, + _mm256_mul_ps(input_values, + _mm256_set1_ps(weights_0[weight_index]))); + accumulators_1 = _mm256_add_ps( + accumulators_1, + _mm256_mul_ps(input_values, + _mm256_set1_ps(weights_1[weight_index]))); + accumulators_2 = _mm256_add_ps( + accumulators_2, + _mm256_mul_ps(input_values, + _mm256_set1_ps(weights_2[weight_index]))); + accumulators_3 = _mm256_add_ps( + accumulators_3, + _mm256_mul_ps(input_values, + _mm256_set1_ps(weights_3[weight_index]))); + } + } + } + _mm256_storeu_ps(output_0 + output_row_offset + output_x, accumulators_0); + _mm256_storeu_ps(output_1 + output_row_offset + output_x, accumulators_1); + _mm256_storeu_ps(output_2 + output_row_offset + output_x, accumulators_2); + _mm256_storeu_ps(output_3 + output_row_offset + output_x, accumulators_3); + output_x += 8u; + } else { + float accumulator_0 = initial_0; + float accumulator_1 = initial_1; + float accumulator_2 = initial_2; + float accumulator_3 = initial_3; + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * input_channel_plane); + const size_t weight_offset = (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const int64_t input_y = (int64_t)output_y * 2 - 1 + kernel_y; + uint32_t kernel_x; + if (input_y < 0 || input_y >= input_height) { + continue; + } + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const int64_t input_x = (int64_t)output_x * 2 - 1 + kernel_x; + if (input_x >= 0 && input_x < input_width) { + const uint64_t input_offset = + (uint64_t)(uint32_t)input_y * input_width + + (uint32_t)input_x; + const float value = + input_channel_data[(size_t)input_offset]; + const size_t weight_index = + weight_offset + (size_t)kernel_y * 3u + kernel_x; + accumulator_0 += value * weights_0[weight_index]; + accumulator_1 += value * weights_1[weight_index]; + accumulator_2 += value * weights_2[weight_index]; + accumulator_3 += value * weights_3[weight_index]; + } + } + } + } + output_0[output_row_offset + output_x] = accumulator_0; + output_1[output_row_offset + output_x] = accumulator_1; + output_2[output_row_offset + output_x] = accumulator_2; + output_3[output_row_offset + output_x] = accumulator_3; + ++output_x; + } + } + } + } + } +} +#endif + +#if LW_COMPILES_AVX2_CONV3X3 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv3x3_stride2_pad1_f32( + const float* input, + const float* weights, + const float* bias, + float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV3X3 + if ((uint32_t)output_dimensions[1] % 8u == 0u) { + conv3x3_stride2_pad1_eight_outputs_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + return; + } + if ((uint32_t)output_dimensions[1] % 4u == 0u) { + conv3x3_stride2_pad1_four_outputs_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + return; + } + if (input_dimensions[1] > 4) { + conv3x3_stride2_pad1_output_stream_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + return; + } + uint32_t input_height = (uint32_t)input_dimensions[2]; + uint32_t input_width = (uint32_t)input_dimensions[3]; + uint32_t output_height = (uint32_t)output_dimensions[2]; + uint32_t output_width = (uint32_t)output_dimensions[3]; + uint64_t input_channel_plane = (uint64_t)input_height * input_width; + uint64_t output_channel_plane = (uint64_t)output_height * output_width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * input_channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * + output_channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * (uint32_t)input_dimensions[1] * 9u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * output_channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + uint32_t output_y; + for (output_y = 0u; output_y < output_height; ++output_y) { + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * output_width); + uint32_t output_x = 0u; + while (output_x < output_width) { + int full_height = output_y != 0u && (uint64_t)output_y * 2u + 1u < input_height; + int full_width = output_x != 0u && output_x + 8u <= output_width && + ((uint64_t)output_x + 7u) * 2u + 1u < input_width; + if (full_height && full_width) { + __m256 accumulators = _mm256_set1_ps(initial); + uint32_t input_channel; + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * input_channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t input_y = output_y * 2u - 1u + kernel_y; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * input_width); + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t input_x = output_x * 2u - 1u + kernel_x; + __m256 first = _mm256_loadu_ps(input_row + input_x); + __m256 second = _mm256_loadu_ps(input_row + input_x + 8u); + __m128 first_even = _mm_shuffle_ps( + _mm256_castps256_ps128(first), + _mm256_extractf128_ps(first, 1), _MM_SHUFFLE(2, 0, 2, 0)); + __m128 second_even = _mm_shuffle_ps( + _mm256_castps256_ps128(second), + _mm256_extractf128_ps(second, 1), _MM_SHUFFLE(2, 0, 2, 0)); + __m256 input_values = _mm256_castps128_ps256(first_even); + __m256 weight_values = _mm256_set1_ps( + weight_channel_data[(size_t)kernel_y * 3u + kernel_x]); + input_values = + _mm256_insertf128_ps(input_values, second_even, 1); + accumulators = _mm256_add_ps( + accumulators, _mm256_mul_ps(input_values, weight_values)); + } + } + } + _mm256_storeu_ps(output_row + output_x, accumulators); + output_x += 8u; + } else { + float accumulator = initial; + uint32_t input_channel; + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * input_channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + int64_t input_y = (int64_t)output_y * 2 - 1 + kernel_y; + uint32_t kernel_x; + if (input_y < 0 || input_y >= input_height) { + continue; + } + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + int64_t input_x = (int64_t)output_x * 2 - 1 + kernel_x; + if (input_x >= 0 && input_x < input_width) { + uint64_t input_offset = + (uint64_t)(uint32_t)input_y * input_width + + (uint32_t)input_x; + accumulator += + input_channel_data[(size_t)input_offset] * + weight_channel_data[(size_t)kernel_y * 3u + kernel_x]; + } + } + } + } + output_row[output_x++] = accumulator; + } + } + } + } + } +#else + lw_scalar_conv3x3_stride2_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); +#endif +} + +#if LW_COMPILES_AVX2_CONV3X3 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_packed_conv3x3_stride2_pad1_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV3X3 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t input_height = (uint32_t)input_dimensions[2]; + const uint32_t input_width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint32_t output_height = (uint32_t)output_dimensions[2]; + const uint32_t output_width = (uint32_t)output_dimensions[3]; + const uint64_t input_plane = (uint64_t)input_height * input_width; + const uint64_t output_plane = (uint64_t)output_height * output_width; + const uint32_t output_blocks = + output_channels / LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * input_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * output_plane); + uint32_t output_block; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + float* output_planes[LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE]; + float initial[LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE]; + uint32_t lane; + for (lane = 0u; lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; ++lane) { + const uint32_t output_channel = + output_block * LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE + lane; + output_planes[lane] = + batch_output + (size_t)((uint64_t)output_channel * output_plane); + initial[lane] = bias == NULL ? 0.0f : bias[output_channel]; + } + { + uint32_t output_y; + for (output_y = 0u; output_y < output_height; ++output_y) { + const size_t output_row_offset = + (size_t)((uint64_t)output_y * output_width); + uint32_t output_x = 0u; + while (output_x < output_width) { + const int full_height = + output_y != 0u && + (uint64_t)output_y * 2u + 1u < input_height; + const int full_width = + output_x != 0u && output_x + 8u <= output_width && + ((uint64_t)output_x + 7u) * 2u + 1u < input_width; + if (full_height && full_width) { + __m256 accumulators[LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE]; + uint32_t input_channel; + for (lane = 0u; + lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; ++lane) { + accumulators[lane] = _mm256_set1_ps(initial[lane]); + } + for (input_channel = 0u; input_channel < input_channels; + ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * input_plane); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const uint32_t input_y = + output_y * 2u - 1u + kernel_y; + const float* input_row = + input_channel_data + + (size_t)((uint64_t)input_y * input_width); + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const uint32_t input_x = + output_x * 2u - 1u + kernel_x; + const uint32_t kernel_index = + kernel_y * 3u + kernel_x; + const float* packed = + packed_weights + + (size_t)((((uint64_t)output_block * + input_channels + + input_channel) * + 9u + + kernel_index) * + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE); + __m256 first = + _mm256_loadu_ps(input_row + input_x); + __m256 second = + _mm256_loadu_ps(input_row + input_x + 8u); + __m128 first_even = _mm_shuffle_ps( + _mm256_castps256_ps128(first), + _mm256_extractf128_ps(first, 1), + _MM_SHUFFLE(2, 0, 2, 0)); + __m128 second_even = _mm_shuffle_ps( + _mm256_castps256_ps128(second), + _mm256_extractf128_ps(second, 1), + _MM_SHUFFLE(2, 0, 2, 0)); + __m256 input_values = + _mm256_castps128_ps256(first_even); + input_values = _mm256_insertf128_ps( + input_values, second_even, 1); + for (lane = 0u; + lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + ++lane) { + accumulators[lane] = _mm256_add_ps( + accumulators[lane], + _mm256_mul_ps( + input_values, + _mm256_set1_ps(packed[lane]))); + } + } + } + } + for (lane = 0u; + lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; ++lane) { + _mm256_storeu_ps( + output_planes[lane] + output_row_offset + output_x, + accumulators[lane]); + } + output_x += 8u; + } else { + float accumulators[LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE]; + uint32_t input_channel; + for (lane = 0u; + lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; ++lane) { + accumulators[lane] = initial[lane]; + } + for (input_channel = 0u; input_channel < input_channels; + ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * input_plane); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const int64_t input_y = + (int64_t)output_y * 2 - 1 + kernel_y; + uint32_t kernel_x; + if (input_y < 0 || + input_y >= (int64_t)input_height) { + continue; + } + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const int64_t input_x = + (int64_t)output_x * 2 - 1 + kernel_x; + if (input_x >= 0 && + input_x < (int64_t)input_width) { + const uint32_t kernel_index = + kernel_y * 3u + kernel_x; + const float* packed = + packed_weights + + (size_t)((((uint64_t)output_block * + input_channels + + input_channel) * + 9u + + kernel_index) * + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE); + const float value = + input_channel_data[(size_t)( + (uint64_t)(uint32_t)input_y * + input_width + + (uint32_t)input_x)]; + for (lane = 0u; + lane < + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + ++lane) { + accumulators[lane] += + value * packed[lane]; + } + } + } + } + } + for (lane = 0u; + lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; ++lane) { + output_planes[lane][output_row_offset + output_x] = + accumulators[lane]; + } + ++output_x; + } + } + } + } + } + } +#else + lw_scalar_packed_conv3x3_stride2_pad1_f32( + input, packed_weights, bias, output, input_dimensions, output_dimensions); +#endif +} + +#if defined(LW_AVX2_FMA_CONV3X3_DISPATCH) +#if LW_COMPILES_AVX2_CONV3X3 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,fma"))) +#endif +void lw_avx2_fma_packed_conv3x3_stride2_pad1_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV3X3 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t input_height = (uint32_t)input_dimensions[2]; + const uint32_t input_width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint32_t output_height = (uint32_t)output_dimensions[2]; + const uint32_t output_width = (uint32_t)output_dimensions[3]; + const uint64_t input_plane = (uint64_t)input_height * input_width; + const uint64_t output_plane = (uint64_t)output_height * output_width; + const uint32_t output_blocks = + output_channels / LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * input_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * output_plane); + uint32_t output_block; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + float* output_planes[LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE]; + float initial[LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE]; + uint32_t lane; + for (lane = 0u; lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; ++lane) { + const uint32_t output_channel = + output_block * LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE + lane; + output_planes[lane] = + batch_output + (size_t)((uint64_t)output_channel * output_plane); + initial[lane] = bias == NULL ? 0.0f : bias[output_channel]; + } + { + uint32_t output_y; + for (output_y = 0u; output_y < output_height; ++output_y) { + const size_t output_row_offset = + (size_t)((uint64_t)output_y * output_width); + uint32_t output_x = 0u; + while (output_x < output_width) { + const int full_height = + output_y != 0u && + (uint64_t)output_y * 2u + 1u < input_height; + const int full_width = + output_x != 0u && output_x + 8u <= output_width && + ((uint64_t)output_x + 7u) * 2u + 1u < input_width; + if (full_height && full_width) { + __m256 accumulators[LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE]; + uint32_t input_channel; + for (lane = 0u; + lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; ++lane) { + accumulators[lane] = _mm256_set1_ps(initial[lane]); + } + for (input_channel = 0u; input_channel < input_channels; + ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * input_plane); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const uint32_t input_y = + output_y * 2u - 1u + kernel_y; + const float* input_row = + input_channel_data + + (size_t)((uint64_t)input_y * input_width); + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const uint32_t input_x = + output_x * 2u - 1u + kernel_x; + const uint32_t kernel_index = + kernel_y * 3u + kernel_x; + const float* packed = + packed_weights + + (size_t)((((uint64_t)output_block * + input_channels + + input_channel) * + 9u + + kernel_index) * + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE); + __m256 first = + _mm256_loadu_ps(input_row + input_x); + __m256 second = + _mm256_loadu_ps(input_row + input_x + 8u); + __m128 first_even = _mm_shuffle_ps( + _mm256_castps256_ps128(first), + _mm256_extractf128_ps(first, 1), + _MM_SHUFFLE(2, 0, 2, 0)); + __m128 second_even = _mm_shuffle_ps( + _mm256_castps256_ps128(second), + _mm256_extractf128_ps(second, 1), + _MM_SHUFFLE(2, 0, 2, 0)); + __m256 input_values = + _mm256_castps128_ps256(first_even); + input_values = _mm256_insertf128_ps( + input_values, second_even, 1); + for (lane = 0u; + lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + ++lane) { + accumulators[lane] = _mm256_fmadd_ps( + input_values, + _mm256_set1_ps(packed[lane]), + accumulators[lane]); + } + } + } + } + for (lane = 0u; + lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; ++lane) { + _mm256_storeu_ps( + output_planes[lane] + output_row_offset + output_x, + accumulators[lane]); + } + output_x += 8u; + } else { + float accumulators[LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE]; + uint32_t input_channel; + for (lane = 0u; + lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; ++lane) { + accumulators[lane] = initial[lane]; + } + for (input_channel = 0u; input_channel < input_channels; + ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * input_plane); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const int64_t input_y = + (int64_t)output_y * 2 - 1 + kernel_y; + uint32_t kernel_x; + if (input_y < 0 || + input_y >= (int64_t)input_height) { + continue; + } + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const int64_t input_x = + (int64_t)output_x * 2 - 1 + kernel_x; + if (input_x >= 0 && + input_x < (int64_t)input_width) { + const uint32_t kernel_index = + kernel_y * 3u + kernel_x; + const float* packed = + packed_weights + + (size_t)((((uint64_t)output_block * + input_channels + + input_channel) * + 9u + + kernel_index) * + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE); + const float value = + input_channel_data[(size_t)( + (uint64_t)(uint32_t)input_y * + input_width + + (uint32_t)input_x)]; + for (lane = 0u; + lane < + LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; + ++lane) { + accumulators[lane] += + value * packed[lane]; + } + } + } + } + } + for (lane = 0u; + lane < LW_PACKED_CONV3X3_STRIDE2_OUTPUT_TILE; ++lane) { + output_planes[lane][output_row_offset + output_x] = + accumulators[lane]; + } + ++output_x; + } + } + } + } + } + } +#else + lw_scalar_packed_conv3x3_stride2_pad1_f32( + input, packed_weights, bias, output, input_dimensions, output_dimensions); +#endif +} +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv3x3_eight.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv3x3_eight.c new file mode 100644 index 00000000..ca91c46a --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv3x3_eight.c @@ -0,0 +1,136 @@ +#include "simd_kernels.h" + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_CONV3X3_EIGHT 1 +#else +# define LW_COMPILES_AVX2_CONV3X3_EIGHT 0 +#endif + +#if LW_COMPILES_AVX2_CONV3X3_EIGHT && defined(LW_EXPERIMENTAL_AVX2_CONV3X3_EIGHT_OUTPUTS) +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +void lw_avx2_conv3x3_unit_pad1_eight_outputs_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t height = (uint32_t)input_dimensions[2]; + const uint32_t width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint64_t channel_plane = (uint64_t)height * width; + const uint64_t weights_per_output = (uint64_t)input_channels * 9u; + uint32_t batch; + + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < output_channels; output_channel += 8u) { + const float* output_weights[8]; + float* output_planes[8]; + float initial[8]; + uint32_t lane; + for (lane = 0u; lane < 8u; ++lane) { + output_weights[lane] = + weights + (size_t)((uint64_t)(output_channel + lane) * weights_per_output); + output_planes[lane] = + batch_output + (size_t)((uint64_t)(output_channel + lane) * channel_plane); + initial[lane] = bias == NULL ? 0.0f : bias[output_channel + lane]; + } + + uint32_t output_y; + for (output_y = 0u; output_y < height; ++output_y) { + const size_t output_row_offset = (size_t)((uint64_t)output_y * width); + uint32_t output_x = 0u; + while (output_x < width) { + const int full_height = output_y != 0u && output_y + 1u < height; + const int full_width = output_x != 0u && output_x + 8u < width; + if (full_height && full_width) { + __m256 accumulators[8]; + uint32_t input_channel; + for (lane = 0u; lane < 8u; ++lane) { + accumulators[lane] = _mm256_set1_ps(initial[lane]); + } + for (input_channel = 0u; input_channel < input_channels; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const size_t weight_offset = (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const uint32_t input_y = output_y + kernel_y - 1u; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const uint32_t input_x = output_x + kernel_x - 1u; + const size_t weight_index = + weight_offset + (size_t)kernel_y * 3u + kernel_x; + const __m256 input_values = + _mm256_loadu_ps(input_row + input_x); + for (lane = 0u; lane < 8u; ++lane) { + accumulators[lane] = _mm256_add_ps( + accumulators[lane], + _mm256_mul_ps( + input_values, + _mm256_set1_ps(output_weights[lane][weight_index]))); + } + } + } + } + for (lane = 0u; lane < 8u; ++lane) { + _mm256_storeu_ps( + output_planes[lane] + output_row_offset + output_x, + accumulators[lane]); + } + output_x += 8u; + } else { + float accumulators[8]; + uint32_t input_channel; + for (lane = 0u; lane < 8u; ++lane) { + accumulators[lane] = initial[lane]; + } + for (input_channel = 0u; input_channel < input_channels; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const size_t weight_offset = (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const int64_t input_y = (int64_t)output_y + kernel_y - 1; + uint32_t kernel_x; + if (input_y < 0 || input_y >= (int64_t)height) { + continue; + } + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const int64_t input_x = (int64_t)output_x + kernel_x - 1; + if (input_x >= 0 && input_x < (int64_t)width) { + const size_t input_index = + (size_t)((uint64_t)(uint32_t)input_y * width + + (uint32_t)input_x); + const size_t weight_index = + weight_offset + (size_t)kernel_y * 3u + kernel_x; + const float value = input_channel_data[input_index]; + for (lane = 0u; lane < 8u; ++lane) { + accumulators[lane] += + value * output_weights[lane][weight_index]; + } + } + } + } + } + for (lane = 0u; lane < 8u; ++lane) { + output_planes[lane][output_row_offset + output_x] = + accumulators[lane]; + } + ++output_x; + } + } + } + } + } +} +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv7x7.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv7x7.c new file mode 100644 index 00000000..d981de28 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_conv7x7.c @@ -0,0 +1,681 @@ +#include "simd_kernels.h" + +#include "scalar_kernels.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_CONV7X7 1 +#else +# define LW_COMPILES_AVX2_CONV7X7 0 +#endif + +#if LW_COMPILES_AVX2_CONV7X7 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv5x5_four_outputs_unit_pad2_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV7X7 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t height = (uint32_t)input_dimensions[2]; + const uint32_t width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint64_t channel_plane = (uint64_t)height * width; + const uint64_t weights_per_output = (uint64_t)input_channels * 25u; + uint32_t batch; + + /* Keep one 8-column tile in registers for the complete input-channel / + * kernel accumulation. Edge pixels use the same scalar order as the + * reference path; interior tiles only change where the partial sum lives. */ + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_base; + for (output_base = 0u; output_base + 4u <= output_channels; output_base += 4u) { + float* output_data[4]; + const float* output_weights[4]; + uint32_t lane; + for (lane = 0u; lane < 4u; ++lane) { + const uint32_t output_channel = output_base + lane; + output_data[lane] = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + output_weights[lane] = + weights + (size_t)((uint64_t)output_channel * weights_per_output); + } + + uint32_t output_y; + for (output_y = 0u; output_y < height; ++output_y) { + const uint32_t kernel_y_begin = output_y < 2u ? 2u - output_y : 0u; + const uint32_t kernel_y_end = + output_y + 2u >= height ? height - output_y + 2u : 5u; + const uint32_t interior_begin = 2u; + const uint32_t interior_end = width > 2u ? width - 2u : 2u; + uint32_t output_x; + + for (output_x = 0u; output_x < interior_begin && output_x < width; ++output_x) { + uint32_t input_channel; + for (lane = 0u; lane < 4u; ++lane) { + float value = bias == NULL ? 0.0f : bias[output_base + lane]; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + for (uint32_t kernel_y = kernel_y_begin; kernel_y < kernel_y_end; + ++kernel_y) { + const uint32_t input_y = output_y + kernel_y - 2u; + for (uint32_t kernel_x = 0u; kernel_x < 5u; ++kernel_x) { + const int32_t input_x = (int32_t)output_x + (int32_t)kernel_x - 2; + if (input_x >= 0 && input_x < (int32_t)width) { + value += input_channel_data[(size_t)((uint64_t)input_y * width + + (uint32_t)input_x)] * + output_weights[lane][input_channel * 25u + + kernel_y * 5u + kernel_x]; + } + } + } + } + output_data[lane][(size_t)((uint64_t)output_y * width + output_x)] = value; + } + } + + for (output_x = interior_begin; output_x + 8u <= interior_end; output_x += 8u) { + __m256 accumulators[4]; + for (lane = 0u; lane < 4u; ++lane) { + accumulators[lane] = _mm256_set1_ps( + bias == NULL ? 0.0f : bias[output_base + lane]); + } + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + for (uint32_t kernel_y = kernel_y_begin; kernel_y < kernel_y_end; + ++kernel_y) { + const uint32_t input_y = output_y + kernel_y - 2u; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + for (uint32_t kernel_x = 0u; kernel_x < 5u; ++kernel_x) { + const __m256 input_values = _mm256_loadu_ps( + input_row + output_x + kernel_x - 2u); + for (lane = 0u; lane < 4u; ++lane) { + const float weight = + output_weights[lane][input_channel * 25u + + kernel_y * 5u + kernel_x]; + accumulators[lane] = _mm256_add_ps( + accumulators[lane], + _mm256_mul_ps(input_values, _mm256_set1_ps(weight))); + } + } + } + } + for (lane = 0u; lane < 4u; ++lane) { + _mm256_storeu_ps(output_data[lane] + (size_t)((uint64_t)output_y * width + + output_x), + accumulators[lane]); + } + } + + for (; output_x < width; ++output_x) { + uint32_t input_channel; + for (lane = 0u; lane < 4u; ++lane) { + float value = bias == NULL ? 0.0f : bias[output_base + lane]; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + for (uint32_t kernel_y = kernel_y_begin; kernel_y < kernel_y_end; + ++kernel_y) { + const uint32_t input_y = output_y + kernel_y - 2u; + for (uint32_t kernel_x = 0u; kernel_x < 5u; ++kernel_x) { + const int32_t input_x = (int32_t)output_x + (int32_t)kernel_x - 2; + if (input_x >= 0 && input_x < (int32_t)width) { + value += input_channel_data[(size_t)((uint64_t)input_y * width + + (uint32_t)input_x)] * + output_weights[lane][input_channel * 25u + + kernel_y * 5u + kernel_x]; + } + } + } + } + output_data[lane][(size_t)((uint64_t)output_y * width + output_x)] = value; + } + } + } + } + } +#else + lw_scalar_conv2d_f32(input, weights, bias, (uint32_t)output_dimensions[1], output, + input_dimensions, (const int32_t[4]){output_dimensions[1], + input_dimensions[1], 5, 5}, + output_dimensions, (const int32_t[2]){5, 5}, + (const int32_t[2]){1, 1}, (const int32_t[2]){1, 1}, + (const int32_t[4]){2, 2, 2, 2}, 1u); +#endif +} + +#if LW_COMPILES_AVX2_CONV7X7 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv7x7_four_outputs_unit_pad3_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV7X7 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t height = (uint32_t)input_dimensions[2]; + const uint32_t width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint64_t channel_plane = (uint64_t)height * width; + const uint64_t weights_per_output = (uint64_t)input_channels * 49u; + uint32_t batch; + + /* Keep one 8-column tile in registers for the complete input-channel / + * kernel accumulation. The scalar border keeps the same edge handling + * as the reference implementation; only interior tiles change where the + * partial sum is stored. */ + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_base; + for (output_base = 0u; output_base + 4u <= output_channels; output_base += 4u) { + float* output_data[4]; + const float* output_weights[4]; + uint32_t lane; + for (lane = 0u; lane < 4u; ++lane) { + const uint32_t output_channel = output_base + lane; + output_data[lane] = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + output_weights[lane] = + weights + (size_t)((uint64_t)output_channel * weights_per_output); + } + + uint32_t output_y; + for (output_y = 0u; output_y < height; ++output_y) { + const uint32_t kernel_y_begin = output_y < 3u ? 3u - output_y : 0u; + const uint32_t kernel_y_end = + output_y + 3u >= height ? height - output_y + 3u : 7u; + const uint32_t interior_begin = 3u; + const uint32_t interior_end = width > 3u ? width - 3u : 3u; + uint32_t output_x; + + for (output_x = 0u; output_x < interior_begin && output_x < width; ++output_x) { + uint32_t input_channel; + for (lane = 0u; lane < 4u; ++lane) { + float value = bias == NULL ? 0.0f : bias[output_base + lane]; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + for (uint32_t kernel_y = kernel_y_begin; kernel_y < kernel_y_end; + ++kernel_y) { + const uint32_t input_y = output_y + kernel_y - 3u; + for (uint32_t kernel_x = 0u; kernel_x < 7u; ++kernel_x) { + const int32_t input_x = + (int32_t)output_x + (int32_t)kernel_x - 3; + if (input_x >= 0 && input_x < (int32_t)width) { + value += input_channel_data[(size_t)((uint64_t)input_y * width + + (uint32_t)input_x)] * + output_weights[lane][input_channel * 49u + + kernel_y * 7u + kernel_x]; + } + } + } + } + output_data[lane][(size_t)((uint64_t)output_y * width + output_x)] = value; + } + } + + for (output_x = interior_begin; output_x + 8u <= interior_end; output_x += 8u) { + __m256 accumulators[4]; + for (lane = 0u; lane < 4u; ++lane) { + accumulators[lane] = _mm256_set1_ps( + bias == NULL ? 0.0f : bias[output_base + lane]); + } + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + for (uint32_t kernel_y = kernel_y_begin; kernel_y < kernel_y_end; + ++kernel_y) { + const uint32_t input_y = output_y + kernel_y - 3u; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + for (uint32_t kernel_x = 0u; kernel_x < 7u; ++kernel_x) { + const __m256 input_values = _mm256_loadu_ps( + input_row + output_x + kernel_x - 3u); + for (lane = 0u; lane < 4u; ++lane) { + const float weight = + output_weights[lane][input_channel * 49u + + kernel_y * 7u + kernel_x]; + accumulators[lane] = _mm256_add_ps( + accumulators[lane], + _mm256_mul_ps(input_values, _mm256_set1_ps(weight))); + } + } + } + } + for (lane = 0u; lane < 4u; ++lane) { + _mm256_storeu_ps(output_data[lane] + (size_t)((uint64_t)output_y * width + + output_x), + accumulators[lane]); + } + } + + for (; output_x < width; ++output_x) { + uint32_t input_channel; + for (lane = 0u; lane < 4u; ++lane) { + float value = bias == NULL ? 0.0f : bias[output_base + lane]; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + for (uint32_t kernel_y = kernel_y_begin; kernel_y < kernel_y_end; + ++kernel_y) { + const uint32_t input_y = output_y + kernel_y - 3u; + for (uint32_t kernel_x = 0u; kernel_x < 7u; ++kernel_x) { + const int32_t input_x = + (int32_t)output_x + (int32_t)kernel_x - 3; + if (input_x >= 0 && input_x < (int32_t)width) { + value += input_channel_data[(size_t)((uint64_t)input_y * width + + (uint32_t)input_x)] * + output_weights[lane][input_channel * 49u + + kernel_y * 7u + kernel_x]; + } + } + } + } + output_data[lane][(size_t)((uint64_t)output_y * width + output_x)] = value; + } + } + } + } + } +#else + lw_scalar_conv2d_f32(input, weights, bias, (uint32_t)output_dimensions[1], output, + input_dimensions, (const int32_t[4]){output_dimensions[1], + input_dimensions[1], 7, 7}, + output_dimensions, (const int32_t[2]){7, 7}, + (const int32_t[2]){1, 1}, (const int32_t[2]){1, 1}, + (const int32_t[4]){3, 3, 3, 3}, 1u); +#endif +} + +#if LW_COMPILES_AVX2_CONV7X7 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv7x7_unit_pad3_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV7X7 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t height = (uint32_t)input_dimensions[2]; + const uint32_t width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint64_t channel_plane = (uint64_t)height * width; + const uint64_t weights_per_output = (uint64_t)input_channels * 49u; + uint32_t batch; + + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < output_channels; ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * weights_per_output); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + const float initial = bias == NULL ? 0.0f : bias[output_channel]; + const __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel; + + for (; spatial + 8u <= channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 49u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 7u; ++kernel_y) { + const uint32_t output_y_begin = kernel_y < 3u ? 3u - kernel_y : 0u; + const uint32_t output_y_end = + kernel_y > 3u ? height - (kernel_y - 3u) : height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 7u; ++kernel_x) { + const uint32_t output_x_begin = kernel_x < 3u ? 3u - kernel_x : 0u; + const uint32_t output_x_end = + kernel_x > 3u ? width - (kernel_x - 3u) : width; + const float weight = weight_channel_data[kernel_y * 7u + kernel_x]; + const __m256 weight_values = _mm256_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + const uint32_t input_y = output_y + kernel_y - 3u; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + const uint32_t input_x = output_x + kernel_x - 3u; + const __m256 input_values = _mm256_loadu_ps(input_row + input_x); + __m256 output_values = _mm256_loadu_ps(output_row + output_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + const uint32_t input_x = output_x + kernel_x - 3u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } + } +#else + lw_scalar_conv2d_f32(input, weights, bias, (uint32_t)output_dimensions[1], output, + input_dimensions, (const int32_t[4]){output_dimensions[1], + input_dimensions[1], 7, 7}, + output_dimensions, (const int32_t[2]){7, 7}, + (const int32_t[2]){1, 1}, (const int32_t[2]){1, 1}, + (const int32_t[4]){3, 3, 3, 3}, 1u); +#endif +} + +#if LW_COMPILES_AVX2_CONV7X7 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv5x5_unit_pad2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV7X7 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t height = (uint32_t)input_dimensions[2]; + const uint32_t width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint64_t channel_plane = (uint64_t)height * width; + const uint64_t weights_per_output = (uint64_t)input_channels * 25u; + uint32_t batch; + + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < output_channels; ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * weights_per_output); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + const float initial = bias == NULL ? 0.0f : bias[output_channel]; + const __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel; + + for (; spatial + 8u <= channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 25u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 5u; ++kernel_y) { + const uint32_t output_y_begin = kernel_y < 2u ? 2u - kernel_y : 0u; + const uint32_t output_y_end = + kernel_y > 2u ? height - (kernel_y - 2u) : height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 5u; ++kernel_x) { + const uint32_t output_x_begin = kernel_x < 2u ? 2u - kernel_x : 0u; + const uint32_t output_x_end = + kernel_x > 2u ? width - (kernel_x - 2u) : width; + const float weight = weight_channel_data[kernel_y * 5u + kernel_x]; + const __m256 weight_values = _mm256_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + const uint32_t input_y = output_y + kernel_y - 2u; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + const uint32_t input_x = output_x + kernel_x - 2u; + const __m256 input_values = _mm256_loadu_ps(input_row + input_x); + __m256 output_values = _mm256_loadu_ps(output_row + output_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + const uint32_t input_x = output_x + kernel_x - 2u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } + } +#else + lw_scalar_conv2d_f32(input, weights, bias, (uint32_t)output_dimensions[1], output, + input_dimensions, (const int32_t[4]){output_dimensions[1], + input_dimensions[1], 5, 5}, + output_dimensions, (const int32_t[2]){5, 5}, + (const int32_t[2]){1, 1}, (const int32_t[2]){1, 1}, + (const int32_t[4]){2, 2, 2, 2}, 1u); +#endif +} + +#if LW_COMPILES_AVX2_CONV7X7 +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static void lw_avx2_conv_axis_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4], uint32_t kernel_size, + uint32_t padding, uint32_t vertical) { +#if LW_COMPILES_AVX2_CONV7X7 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t height = (uint32_t)input_dimensions[2]; + const uint32_t width = (uint32_t)input_dimensions[3]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint64_t channel_plane = (uint64_t)height * width; + const uint64_t weights_per_output = (uint64_t)input_channels * kernel_size; + uint32_t batch; + + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < output_channels; ++output_channel) { + const float* output_weights = + weights + (size_t)((uint64_t)output_channel * weights_per_output); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + const float initial = bias == NULL ? 0.0f : bias[output_channel]; + const __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel; + + for (; spatial + 8u <= channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* channel_weights = + output_weights + (size_t)((uint64_t)input_channel * kernel_size); + uint32_t kernel_index; + for (kernel_index = 0u; kernel_index < kernel_size; ++kernel_index) { + const float weight = channel_weights[kernel_index]; + const __m256 weight_values = _mm256_set1_ps(weight); + uint32_t output_y; + if (vertical != 0u) { + const uint32_t output_y_begin = + kernel_index < padding ? padding - kernel_index : 0u; + const uint32_t output_y_end = + kernel_index > padding ? height - (kernel_index - padding) : height; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + const uint32_t input_y = output_y + kernel_index - padding; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x = 0u; + for (; output_x + 8u <= width; output_x += 8u) { + const __m256 input_values = _mm256_loadu_ps(input_row + output_x); + __m256 output_values = _mm256_loadu_ps(output_row + output_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < width; ++output_x) { + output_row[output_x] += input_row[output_x] * weight; + } + } + } else { + const uint32_t output_x_begin = + kernel_index < padding ? padding - kernel_index : 0u; + const uint32_t output_x_end = + kernel_index > padding ? width - (kernel_index - padding) : width; + for (output_y = 0u; output_y < height; ++output_y) { + const float* input_row = + input_channel_data + (size_t)((uint64_t)output_y * width); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + const uint32_t input_x = output_x + kernel_index - padding; + const __m256 input_values = _mm256_loadu_ps(input_row + input_x); + __m256 output_values = _mm256_loadu_ps(output_row + output_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + const uint32_t input_x = output_x + kernel_index - padding; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } + } +#else + (void)input; + (void)weights; + (void)bias; + (void)output; + (void)input_dimensions; + (void)output_dimensions; + (void)kernel_size; + (void)padding; + (void)vertical; +#endif +} +#endif + +#if LW_COMPILES_AVX2_CONV7X7 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv7x1_unit_pad3_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV7X7 + lw_avx2_conv_axis_f32(input, weights, bias, output, input_dimensions, output_dimensions, 7u, + 3u, 1u); +#else + lw_scalar_conv2d_f32(input, weights, bias, (uint32_t)output_dimensions[1], output, + input_dimensions, (const int32_t[4]){output_dimensions[1], + input_dimensions[1], 7, 1}, + output_dimensions, (const int32_t[2]){7, 1}, + (const int32_t[2]){1, 1}, (const int32_t[2]){1, 1}, + (const int32_t[4]){3, 0, 3, 0}, 1u); +#endif +} + +#if LW_COMPILES_AVX2_CONV7X7 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv1x7_unit_pad3_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV7X7 + lw_avx2_conv_axis_f32(input, weights, bias, output, input_dimensions, output_dimensions, 7u, + 3u, 0u); +#else + lw_scalar_conv2d_f32(input, weights, bias, (uint32_t)output_dimensions[1], output, + input_dimensions, (const int32_t[4]){output_dimensions[1], + input_dimensions[1], 1, 7}, + output_dimensions, (const int32_t[2]){1, 7}, + (const int32_t[2]){1, 1}, (const int32_t[2]){1, 1}, + (const int32_t[4]){0, 3, 0, 3}, 1u); +#endif +} + +#if LW_COMPILES_AVX2_CONV7X7 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv5x1_unit_pad2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV7X7 + lw_avx2_conv_axis_f32(input, weights, bias, output, input_dimensions, output_dimensions, 5u, + 2u, 1u); +#else + lw_scalar_conv2d_f32(input, weights, bias, (uint32_t)output_dimensions[1], output, + input_dimensions, (const int32_t[4]){output_dimensions[1], + input_dimensions[1], 5, 1}, + output_dimensions, (const int32_t[2]){5, 1}, + (const int32_t[2]){1, 1}, (const int32_t[2]){1, 1}, + (const int32_t[4]){2, 0, 2, 0}, 1u); +#endif +} + +#if LW_COMPILES_AVX2_CONV7X7 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv1x5_unit_pad2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_CONV7X7 + lw_avx2_conv_axis_f32(input, weights, bias, output, input_dimensions, output_dimensions, 5u, + 2u, 0u); +#else + lw_scalar_conv2d_f32(input, weights, bias, (uint32_t)output_dimensions[1], output, + input_dimensions, (const int32_t[4]){output_dimensions[1], + input_dimensions[1], 1, 5}, + output_dimensions, (const int32_t[2]){1, 5}, + (const int32_t[2]){1, 1}, (const int32_t[2]){1, 1}, + (const int32_t[4]){0, 2, 0, 2}, 1u); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_convtranspose2x2.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_convtranspose2x2.c new file mode 100644 index 00000000..61e3e313 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_convtranspose2x2.c @@ -0,0 +1,153 @@ +#include "simd_kernels.h" + +#include "scalar_kernels.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_CONV_TRANSPOSE2X2 1 +#else +# define LW_COMPILES_AVX2_CONV_TRANSPOSE2X2 0 +#endif + +#if LW_COMPILES_AVX2_CONV_TRANSPOSE2X2 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_conv_transpose2x2_stride2_range_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4], + uint32_t output_channel_begin, uint32_t output_channel_end) { +#if LW_COMPILES_AVX2_CONV_TRANSPOSE2X2 + uint32_t input_channels = (uint32_t)input_dimensions[1]; + uint32_t output_channels = (uint32_t)output_dimensions[1]; + uint32_t input_height = (uint32_t)input_dimensions[2]; + uint32_t input_width = (uint32_t)input_dimensions[3]; + uint32_t output_width = (uint32_t)output_dimensions[3]; + uint64_t input_plane = (uint64_t)input_height * input_width; + uint64_t output_plane = (uint64_t)(uint32_t)output_dimensions[2] * output_width; + uint32_t batch; + + /* With kernel=2, stride=2 and zero padding, each input point writes one + * non-overlapping 2x2 output tile. Eight adjacent input values expand to + * two contiguous AVX output vectors per tile row, avoiding gathers and + * preserving each output element's input-channel accumulation order. */ + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * input_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * output_plane); + uint32_t output_channel; + for (output_channel = output_channel_begin; output_channel < output_channel_end; + ++output_channel) { + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * output_plane); + __m256 initial = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_channel]); + const float initial_scalar = bias == NULL ? 0.0f : bias[output_channel]; + const __m256 zero = _mm256_setzero_ps(); + uint32_t input_channel; + + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * input_plane); + const float* channel_weights = + weights + (size_t)(((uint64_t)input_channel * output_channels + output_channel) * 4u); + const __m256 weight0 = _mm256_set1_ps(channel_weights[0]); + const __m256 weight1 = _mm256_set1_ps(channel_weights[1]); + const __m256 weight2 = _mm256_set1_ps(channel_weights[2]); + const __m256 weight3 = _mm256_set1_ps(channel_weights[3]); + uint32_t input_y; + for (input_y = 0u; input_y < input_height; ++input_y) { + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * input_width); + float* output_row0 = + output_channel_data + (size_t)((uint64_t)(input_y * 2u) * output_width); + float* output_row1 = output_row0 + output_width; + uint32_t input_x = 0u; + for (; input_x + 8u <= input_width; input_x += 8u) { + __m256 values = _mm256_loadu_ps(input_row + input_x); + __m256 even_lo_pairs = _mm256_unpacklo_ps(values, zero); + __m256 even_hi_pairs = _mm256_unpackhi_ps(values, zero); + __m256 odd_lo_pairs = _mm256_unpacklo_ps(zero, values); + __m256 odd_hi_pairs = _mm256_unpackhi_ps(zero, values); + /* AVX unpack is lane-local: combine the two 128-bit + * halves so vectors hold x[0..3] then x[4..7]. */ + __m256 even_low = + _mm256_permute2f128_ps(even_lo_pairs, even_hi_pairs, 0x20); + __m256 even_high = + _mm256_permute2f128_ps(even_lo_pairs, even_hi_pairs, 0x31); + __m256 odd_low = + _mm256_permute2f128_ps(odd_lo_pairs, odd_hi_pairs, 0x20); + __m256 odd_high = + _mm256_permute2f128_ps(odd_lo_pairs, odd_hi_pairs, 0x31); + uint32_t output_x = input_x * 2u; + __m256 row0_low = input_channel == 0u + ? initial + : _mm256_loadu_ps(output_row0 + output_x); + __m256 row0_high = input_channel == 0u + ? initial + : _mm256_loadu_ps(output_row0 + output_x + 8u); + __m256 row1_low = input_channel == 0u + ? initial + : _mm256_loadu_ps(output_row1 + output_x); + __m256 row1_high = input_channel == 0u + ? initial + : _mm256_loadu_ps(output_row1 + output_x + 8u); + + row0_low = + _mm256_add_ps(row0_low, _mm256_mul_ps(even_low, weight0)); + row0_high = + _mm256_add_ps(row0_high, _mm256_mul_ps(even_high, weight0)); + row0_low = _mm256_add_ps(row0_low, _mm256_mul_ps(odd_low, weight1)); + row0_high = _mm256_add_ps(row0_high, _mm256_mul_ps(odd_high, weight1)); + row1_low = + _mm256_add_ps(row1_low, _mm256_mul_ps(even_low, weight2)); + row1_high = + _mm256_add_ps(row1_high, _mm256_mul_ps(even_high, weight2)); + row1_low = _mm256_add_ps(row1_low, _mm256_mul_ps(odd_low, weight3)); + row1_high = _mm256_add_ps(row1_high, _mm256_mul_ps(odd_high, weight3)); + + _mm256_storeu_ps(output_row0 + output_x, row0_low); + _mm256_storeu_ps(output_row0 + output_x + 8u, row0_high); + _mm256_storeu_ps(output_row1 + output_x, row1_low); + _mm256_storeu_ps(output_row1 + output_x + 8u, row1_high); + } + for (; input_x < input_width; ++input_x) { + float value = input_row[input_x]; + uint32_t output_x = input_x * 2u; + float row00 = + input_channel == 0u ? initial_scalar : output_row0[output_x]; + float row01 = + input_channel == 0u ? initial_scalar : output_row0[output_x + 1u]; + float row10 = + input_channel == 0u ? initial_scalar : output_row1[output_x]; + float row11 = + input_channel == 0u ? initial_scalar : output_row1[output_x + 1u]; + output_row0[output_x] = row00 + value * channel_weights[0]; + output_row0[output_x + 1u] = row01 + value * channel_weights[1]; + output_row1[output_x] = row10 + value * channel_weights[2]; + output_row1[output_x + 1u] = row11 + value * channel_weights[3]; + } + } + } + } + } +#else + (void)input; + (void)weights; + (void)bias; + (void)output; + (void)input_dimensions; + (void)output_dimensions; + (void)output_channel_begin; + (void)output_channel_end; +#endif +} + +void lw_avx2_conv_transpose2x2_stride2_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { + lw_avx2_conv_transpose2x2_stride2_range_f32( + input, weights, bias, output, input_dimensions, output_dimensions, 0u, + (uint32_t)output_dimensions[1]); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_depthwise3x3.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_depthwise3x3.c new file mode 100644 index 00000000..9292dede --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_depthwise3x3.c @@ -0,0 +1,157 @@ +#include "simd_kernels.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_DEPTHWISE3X3 1 +#else +# define LW_COMPILES_AVX2_DEPTHWISE3X3 0 +#endif + +#if LW_COMPILES_AVX2_DEPTHWISE3X3 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_depthwise_conv3x3_unit_pad1_f32( + const float* input, + const float* weights, + const float* bias, + float* output, + const int32_t dimensions[4]) { +#if LW_COMPILES_AVX2_DEPTHWISE3X3 + uint32_t height = (uint32_t)dimensions[2]; + uint32_t width = (uint32_t)dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)dimensions[0]; ++batch) { + uint32_t channel; + for (channel = 0u; channel < (uint32_t)dimensions[1]; ++channel) { + uint64_t channel_offset = + ((uint64_t)batch * (uint32_t)dimensions[1] + channel) * channel_plane; + const float* input_channel = input + (size_t)channel_offset; + const float* channel_weights = weights + (size_t)channel * 9u; + float* output_channel = output + (size_t)channel_offset; + float initial = bias == NULL ? 0.0f : bias[channel]; + __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t kernel_y; + for (; spatial + 8u <= channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel[(size_t)spatial] = initial; + } + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = kernel_y == 2u && height != 0u ? height - 1u : height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = kernel_x == 2u && width != 0u ? width - 1u : width; + float weight = channel_weights[kernel_y * 3u + kernel_x]; + __m256 weight_values = _mm256_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y + kernel_y - 1u; + const float* input_row = + input_channel + (size_t)((uint64_t)input_y * width); + float* output_row = output_channel + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + uint32_t input_x = output_x + kernel_x - 1u; + __m256 output_values = _mm256_loadu_ps(output_row + output_x); + __m256 input_values = _mm256_loadu_ps(input_row + input_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 1u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } +#else + lw_scalar_depthwise_conv3x3_unit_pad1_f32(input, weights, bias, output, dimensions); +#endif +} + +#if LW_COMPILES_AVX2_DEPTHWISE3X3 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_depthwise_conv3x3_stride2x1_pad1_f32( + const float* input, + const float* weights, + const float* bias, + float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_DEPTHWISE3X3 + uint32_t input_height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint32_t output_height = (uint32_t)output_dimensions[2]; + uint64_t input_channel_plane = (uint64_t)input_height * width; + uint64_t output_channel_plane = (uint64_t)output_height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + uint32_t channel; + for (channel = 0u; channel < (uint32_t)input_dimensions[1]; ++channel) { + uint64_t input_channel_offset = + ((uint64_t)batch * (uint32_t)input_dimensions[1] + channel) * input_channel_plane; + uint64_t output_channel_offset = + ((uint64_t)batch * (uint32_t)output_dimensions[1] + channel) * + output_channel_plane; + const float* input_channel = input + (size_t)input_channel_offset; + const float* channel_weights = weights + (size_t)channel * 9u; + float* output_channel = output + (size_t)output_channel_offset; + float initial = bias == NULL ? 0.0f : bias[channel]; + __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t kernel_y; + for (; spatial + 8u <= output_channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel + (size_t)spatial, initial_values); + } + for (; spatial < output_channel_plane; ++spatial) { + output_channel[(size_t)spatial] = initial; + } + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = kernel_y == 2u ? input_height / 2u : output_height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = kernel_x == 2u && width != 0u ? width - 1u : width; + float weight = channel_weights[kernel_y * 3u + kernel_x]; + __m256 weight_values = _mm256_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y * 2u + kernel_y - 1u; + const float* input_row = + input_channel + (size_t)((uint64_t)input_y * width); + float* output_row = output_channel + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + uint32_t input_x = output_x + kernel_x - 1u; + __m256 output_values = _mm256_loadu_ps(output_row + output_x); + __m256 input_values = _mm256_loadu_ps(input_row + input_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 1u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } +#else + lw_scalar_depthwise_conv3x3_stride2x1_pad1_f32( + input, weights, bias, output, input_dimensions, output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_depthwise5x5.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_depthwise5x5.c new file mode 100644 index 00000000..16e7324e --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_depthwise5x5.c @@ -0,0 +1,81 @@ +#include "simd_kernels.h" + +#include "scalar_kernels.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_DEPTHWISE5X5 1 +#else +# define LW_COMPILES_AVX2_DEPTHWISE5X5 0 +#endif + +#if LW_COMPILES_AVX2_DEPTHWISE5X5 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_depthwise_conv5x5_unit_pad2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]) { +#if LW_COMPILES_AVX2_DEPTHWISE5X5 + uint32_t height = (uint32_t)dimensions[2]; + uint32_t width = (uint32_t)dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)dimensions[0]; ++batch) { + uint32_t channel; + for (channel = 0u; channel < (uint32_t)dimensions[1]; ++channel) { + uint64_t channel_offset = + ((uint64_t)batch * (uint32_t)dimensions[1] + channel) * channel_plane; + const float* input_channel = input + (size_t)channel_offset; + const float* channel_weights = weights + (size_t)channel * 25u; + float* output_channel = output + (size_t)channel_offset; + float initial = bias == NULL ? 0.0f : bias[channel]; + __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t kernel_y; + for (; spatial + 8u <= channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel[(size_t)spatial] = initial; + } + for (kernel_y = 0u; kernel_y < 5u; ++kernel_y) { + uint32_t output_y_begin = kernel_y < 2u ? 2u - kernel_y : 0u; + uint32_t output_y_trim = kernel_y > 2u ? kernel_y - 2u : 0u; + uint32_t output_y_end = output_y_trim < height ? height - output_y_trim : 0u; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 5u; ++kernel_x) { + uint32_t output_x_begin = kernel_x < 2u ? 2u - kernel_x : 0u; + uint32_t output_x_trim = kernel_x > 2u ? kernel_x - 2u : 0u; + uint32_t output_x_end = output_x_trim < width ? width - output_x_trim : 0u; + float weight = channel_weights[kernel_y * 5u + kernel_x]; + __m256 weight_values = _mm256_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y + kernel_y - 2u; + const float* input_row = + input_channel + (size_t)((uint64_t)input_y * width); + float* output_row = output_channel + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + uint32_t input_x = output_x + kernel_x - 2u; + __m256 input_values = _mm256_loadu_ps(input_row + input_x); + __m256 output_values = _mm256_loadu_ps(output_row + output_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 2u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } +#else + lw_scalar_depthwise_conv5x5_unit_pad2_f32(input, weights, bias, output, dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_depthwise9x9.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_depthwise9x9.c new file mode 100644 index 00000000..e1906651 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_depthwise9x9.c @@ -0,0 +1,94 @@ +#include "simd_kernels.h" + +#include "scalar_kernels.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_DEPTHWISE9X9 1 +#else +# define LW_COMPILES_AVX2_DEPTHWISE9X9 0 +#endif + +#if LW_COMPILES_AVX2_DEPTHWISE9X9 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_depthwise_conv9x9_unit_pad4_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]) { +#if LW_COMPILES_AVX2_DEPTHWISE9X9 + const uint32_t height = (uint32_t)dimensions[2]; + const uint32_t width = (uint32_t)dimensions[3]; + const uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + + for (batch = 0u; batch < (uint32_t)dimensions[0]; ++batch) { + uint32_t channel; + for (channel = 0u; channel < (uint32_t)dimensions[1]; ++channel) { + const uint64_t channel_offset = + ((uint64_t)batch * (uint32_t)dimensions[1] + channel) * channel_plane; + const float* input_channel = input + (size_t)channel_offset; + const float* channel_weights = weights + (size_t)channel * 81u; + float* output_channel = output + (size_t)channel_offset; + const float initial = bias == NULL ? 0.0f : bias[channel]; + const __m256 initial_values = _mm256_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t kernel_y; + + for (; spatial + 8u <= channel_plane; spatial += 8u) { + _mm256_storeu_ps(output_channel + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel[(size_t)spatial] = initial; + } + + for (kernel_y = 0u; kernel_y < 9u; ++kernel_y) { + const uint32_t output_y_begin = kernel_y < 4u ? 4u - kernel_y : 0u; + const uint32_t output_y_trim = kernel_y > 4u ? kernel_y - 4u : 0u; + const uint32_t output_y_end = + output_y_trim < height ? height - output_y_trim : 0u; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 9u; ++kernel_x) { + const uint32_t output_x_begin = kernel_x < 4u ? 4u - kernel_x : 0u; + const uint32_t output_x_trim = kernel_x > 4u ? kernel_x - 4u : 0u; + const uint32_t output_x_end = + output_x_trim < width ? width - output_x_trim : 0u; + const float weight = channel_weights[kernel_y * 9u + kernel_x]; + const __m256 weight_values = _mm256_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + const uint32_t input_y = output_y + kernel_y - 4u; + const float* input_row = + input_channel + (size_t)((uint64_t)input_y * width); + float* output_row = + output_channel + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 8u <= output_x_end; output_x += 8u) { + const uint32_t input_x = output_x + kernel_x - 4u; + const __m256 input_values = _mm256_loadu_ps(input_row + input_x); + __m256 output_values = _mm256_loadu_ps(output_row + output_x); + output_values = _mm256_add_ps( + output_values, _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + const uint32_t input_x = output_x + kernel_x - 4u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } +#else + const int32_t weight_dimensions[4] = {dimensions[1], 1, 9, 9}; + const int32_t kernel[2] = {9, 9}; + const int32_t strides[2] = {1, 1}; + const int32_t dilations[2] = {1, 1}; + const int32_t pads[4] = {4, 4, 4, 4}; + (void)lw_scalar_conv2d_f32(input, weights, bias, (uint32_t)dimensions[1], output, dimensions, + weight_dimensions, dimensions, kernel, strides, dilations, pads, + (uint32_t)dimensions[1]); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_erf.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_erf.c new file mode 100644 index 00000000..785ae2a4 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_erf.c @@ -0,0 +1,195 @@ +#include "simd_kernels.h" +#include "nhwc_internal.h" + +/* AVX2 Erf approximation used by GELU-shaped model graphs. */ + +#include +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_ERF 1 +#else +# define LW_COMPILES_AVX2_ERF 0 +#endif + +#if LW_COMPILES_AVX2_ERF +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static __m256 +erf_small_magnitude_f32(__m256 absolute, __m256 squared) { + __m256 polynomial = _mm256_set1_ps(1.0590875083315439e-6f); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-1.3906452410711274e-5f), _mm256_mul_ps(squared, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.1955437252428243e-4f), _mm256_mul_ps(squared, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-8.5424759600797662e-4f), _mm256_mul_ps(squared, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(5.2237718994271529e-3f), _mm256_mul_ps(squared, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-2.6866128888288668e-2f), _mm256_mul_ps(squared, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.1283791226625459e-1f), _mm256_mul_ps(squared, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-3.7612638883043881e-1f), _mm256_mul_ps(squared, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.1283791670929921f), _mm256_mul_ps(squared, polynomial)); + return _mm256_mul_ps(absolute, polynomial); +} + +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static __m256 +erf_middle_magnitude_f32(__m256 absolute) { + __m256 shifted = _mm256_sub_ps(absolute, _mm256_set1_ps(1.5f)); + __m256 polynomial = _mm256_set1_ps(-2.4006675278365739e-3f); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-3.8855162788028288e-3f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.9332860298601401e-2f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-1.5013607234871428e-2f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-4.4599369472787913e-2f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.3876136033191724e-1f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-1.7839541988688759e-1f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.1893013063163335e-1f), _mm256_mul_ps(shifted, polynomial)); + return _mm256_add_ps(_mm256_set1_ps(9.6610514641406819e-1f), + _mm256_mul_ps(shifted, polynomial)); +} + +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static __m256 +erf_large_magnitude_f32(__m256 absolute) { + __m256 shifted = _mm256_sub_ps(absolute, _mm256_set1_ps(3.0f)); + __m256 polynomial = _mm256_set1_ps(-8.8750765320565021e-5f); + polynomial = + _mm256_add_ps(_mm256_set1_ps(3.8805285630072217e-4f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-7.7812010711428433e-4f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.0255980996254569e-3f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-1.0307062838910627e-3f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(7.8586080129729565e-4f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(-4.1936053855221229e-4f), _mm256_mul_ps(shifted, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.3951109720745927e-4f), _mm256_mul_ps(shifted, polynomial)); + return _mm256_add_ps(_mm256_set1_ps(9.9997793886838715e-1f), + _mm256_mul_ps(shifted, polynomial)); +} + +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static __m256 +erf_approximation_f32(__m256 value) { + const __m256 absolute_mask = _mm256_castsi256_ps(_mm256_set1_epi32((int)UINT32_C(0x7fffffff))); + const __m256 sign_mask = _mm256_castsi256_ps(_mm256_set1_epi32((int)UINT32_C(0x80000000))); + const __m256 one = _mm256_set1_ps(1.0f); + __m256 absolute = _mm256_and_ps(value, absolute_mask); + __m256 squared = _mm256_mul_ps(absolute, absolute); + __m256 small = erf_small_magnitude_f32(absolute, squared); + __m256 middle = erf_middle_magnitude_f32(absolute); + __m256 large = erf_large_magnitude_f32(absolute); + __m256 mask; + __m256 result; + mask = _mm256_cmp_ps(absolute, _mm256_set1_ps(2.0f), _CMP_LT_OQ); + result = _mm256_blendv_ps(large, middle, mask); + mask = _mm256_cmp_ps(absolute, one, _CMP_LT_OQ); + result = _mm256_blendv_ps(result, small, mask); + mask = _mm256_cmp_ps(absolute, _mm256_set1_ps(4.0f), _CMP_GE_OQ); + result = _mm256_blendv_ps(result, one, mask); + result = _mm256_xor_ps(result, _mm256_and_ps(value, sign_mask)); + mask = _mm256_cmp_ps(value, value, _CMP_UNORD_Q); + return _mm256_blendv_ps(result, value, mask); +} +#endif + +#if LW_COMPILES_AVX2_ERF && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_erf_f32(const float* input, float* output, uint64_t element_count) { +#if LW_COMPILES_AVX2_ERF + uint64_t index = 0u; + for (; index + 8u <= element_count; index += 8u) { + __m256 value = _mm256_loadu_ps(input + (size_t)index); + _mm256_storeu_ps(output + (size_t)index, erf_approximation_f32(value)); + } + for (; index < element_count; ++index) { + output[(size_t)index] = erff(input[(size_t)index]); + } +#else + uint64_t index; + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = erff(input[(size_t)index]); + } +#endif +} + +/* Exact per-element GELU helpers for the fused convolution epilogues. They + * share erf_approximation_f32 with lw_avx2_gelu_f32 below, so the fused path + * stays bit-identical to running the standalone pass on the stored tensor. + * The vector helper must not be inlined into FMA-enabled callers: the + * polynomial relies on separate mul/add rounding, so it carries a noinline + * attribute in addition to the no-fma target. */ +float lw_nhwc_gelu_scalar_exact_f32(float value) { + float scaled = value / 1.4142135381698608f; + float activated = erff(scaled) + 1.0f; + activated = value * activated; + return activated * 0.5f; +} + +#if LW_COMPILES_AVX2_ERF +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"), noinline)) +# elif defined(_MSC_VER) +__declspec(noinline) +# endif +__m256 lw_nhwc_avx2_gelu_vector_exact_f32(__m256 value) { + const __m256 square_root_two = _mm256_set1_ps(1.4142135381698608f); + const __m256 one = _mm256_set1_ps(1.0f); + const __m256 half = _mm256_set1_ps(0.5f); + __m256 scaled = _mm256_div_ps(value, square_root_two); + __m256 activated = _mm256_add_ps(erf_approximation_f32(scaled), one); + activated = _mm256_mul_ps(value, activated); + return _mm256_mul_ps(activated, half); +} +#endif + +#if LW_COMPILES_AVX2_ERF && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_gelu_f32(const float* input, float* output, uint64_t element_count) { +#if LW_COMPILES_AVX2_ERF + const __m256 square_root_two = _mm256_set1_ps(1.4142135381698608f); + const __m256 one = _mm256_set1_ps(1.0f); + const __m256 half = _mm256_set1_ps(0.5f); + uint64_t index = 0u; + for (; index + 8u <= element_count; index += 8u) { + __m256 value = _mm256_loadu_ps(input + (size_t)index); + __m256 scaled = _mm256_div_ps(value, square_root_two); + __m256 activated = _mm256_add_ps(erf_approximation_f32(scaled), one); + activated = _mm256_mul_ps(value, activated); + _mm256_storeu_ps(output + (size_t)index, _mm256_mul_ps(activated, half)); + } + for (; index < element_count; ++index) { + output[(size_t)index] = lw_nhwc_gelu_scalar_exact_f32(input[(size_t)index]); + } +#else + uint64_t index; + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = lw_nhwc_gelu_scalar_exact_f32(input[(size_t)index]); + } +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_fma_packed_conv1x1.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_fma_packed_conv1x1.c new file mode 100644 index 00000000..9a3239c4 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_fma_packed_conv1x1.c @@ -0,0 +1,194 @@ +#include "simd_kernels.h" + +#include "packed_conv_internal.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_PACKED_CONV1X1 1 +#else +# define LW_COMPILES_AVX2_PACKED_CONV1X1 0 +#endif + +#if LW_COMPILES_AVX2_PACKED_CONV1X1 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,fma"))) +#endif +void lw_avx2_fma_packed_conv1x1_f32(const float* input, const float* packed_weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_PACKED_CONV1X1 + uint32_t input_channels = (uint32_t)input_dimensions[1]; + uint32_t output_channels = (uint32_t)output_dimensions[1]; + uint64_t channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + uint32_t output_blocks = + (output_channels + LW_PACKED_CONV1X1_OUTPUT_TILE - 1u) / LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_block; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + uint32_t output_base = output_block * LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t valid_outputs = output_channels - output_base; + uint64_t spatial = 0u; + if (valid_outputs > LW_PACKED_CONV1X1_OUTPUT_TILE) { + valid_outputs = LW_PACKED_CONV1X1_OUTPUT_TILE; + } +# if defined(_M_X64) || defined(__x86_64__) + /* x64 exposes sixteen YMM registers, so two spatial vectors can + * stay live for each of the four output planes. This halves the + * packed-weight broadcasts and loop bookkeeping compared with the + * 4x8 kernel below. x86 retains 4x8 because it has only eight + * architectural vector registers and would otherwise spill. */ + if (valid_outputs == LW_PACKED_CONV1X1_OUTPUT_TILE) { + for (; spatial + 16u <= channel_plane; spatial += 16u) { + __m256 accumulator0_low = + _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base]); + __m256 accumulator0_high = accumulator0_low; + __m256 accumulator1_low = + _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 1u]); + __m256 accumulator1_high = accumulator1_low; + __m256 accumulator2_low = + _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 2u]); + __m256 accumulator2_high = accumulator2_low; + __m256 accumulator3_low = + _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 3u]); + __m256 accumulator3_high = accumulator3_low; + const float* input_ptr = batch_input + (size_t)spatial; + const float* packed_ptr = + packed_weights + + (size_t)((uint64_t)output_block * input_channels * + LW_PACKED_CONV1X1_OUTPUT_TILE); + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + __m256 input_low = _mm256_loadu_ps(input_ptr); + __m256 input_high = _mm256_loadu_ps(input_ptr + 8u); + __m256 weight = _mm256_set1_ps(packed_ptr[0]); + accumulator0_low = + _mm256_fmadd_ps(input_low, weight, accumulator0_low); + accumulator0_high = + _mm256_fmadd_ps(input_high, weight, accumulator0_high); + weight = _mm256_set1_ps(packed_ptr[1]); + accumulator1_low = + _mm256_fmadd_ps(input_low, weight, accumulator1_low); + accumulator1_high = + _mm256_fmadd_ps(input_high, weight, accumulator1_high); + weight = _mm256_set1_ps(packed_ptr[2]); + accumulator2_low = + _mm256_fmadd_ps(input_low, weight, accumulator2_low); + accumulator2_high = + _mm256_fmadd_ps(input_high, weight, accumulator2_high); + weight = _mm256_set1_ps(packed_ptr[3]); + accumulator3_low = + _mm256_fmadd_ps(input_low, weight, accumulator3_low); + accumulator3_high = + _mm256_fmadd_ps(input_high, weight, accumulator3_high); + input_ptr += (size_t)channel_plane; + packed_ptr += LW_PACKED_CONV1X1_OUTPUT_TILE; + } + float* output0 = + batch_output + (size_t)((uint64_t)output_base * channel_plane + spatial); + float* output1 = output0 + (size_t)channel_plane; + float* output2 = output1 + (size_t)channel_plane; + float* output3 = output2 + (size_t)channel_plane; + _mm256_storeu_ps(output0, accumulator0_low); + _mm256_storeu_ps(output0 + 8u, accumulator0_high); + _mm256_storeu_ps(output1, accumulator1_low); + _mm256_storeu_ps(output1 + 8u, accumulator1_high); + _mm256_storeu_ps(output2, accumulator2_low); + _mm256_storeu_ps(output2 + 8u, accumulator2_high); + _mm256_storeu_ps(output3, accumulator3_low); + _mm256_storeu_ps(output3 + 8u, accumulator3_high); + } + } +# endif + for (; spatial + 8u <= channel_plane; spatial += 8u) { + __m256 accumulator0 = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base]); + __m256 accumulator1 = _mm256_set1_ps( + bias == NULL || valid_outputs <= 1u ? 0.0f : bias[output_base + 1u]); + __m256 accumulator2 = _mm256_set1_ps( + bias == NULL || valid_outputs <= 2u ? 0.0f : bias[output_base + 2u]); + __m256 accumulator3 = _mm256_set1_ps( + bias == NULL || valid_outputs <= 3u ? 0.0f : bias[output_base + 3u]); + const float* input_ptr = batch_input + (size_t)spatial; + const float* packed_ptr = + packed_weights + + (size_t)((uint64_t)output_block * input_channels * + LW_PACKED_CONV1X1_OUTPUT_TILE); + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + __m256 input_values = _mm256_loadu_ps(input_ptr); + accumulator0 = _mm256_fmadd_ps( + input_values, _mm256_set1_ps(packed_ptr[0]), accumulator0); + accumulator1 = _mm256_fmadd_ps( + input_values, _mm256_set1_ps(packed_ptr[1]), accumulator1); + accumulator2 = _mm256_fmadd_ps( + input_values, _mm256_set1_ps(packed_ptr[2]), accumulator2); + accumulator3 = _mm256_fmadd_ps( + input_values, _mm256_set1_ps(packed_ptr[3]), accumulator3); + input_ptr += (size_t)channel_plane; + packed_ptr += LW_PACKED_CONV1X1_OUTPUT_TILE; + } + float* output0 = + batch_output + (size_t)((uint64_t)output_base * channel_plane + spatial); + float* output1 = output0 + (size_t)channel_plane; + float* output2 = output1 + (size_t)channel_plane; + float* output3 = output2 + (size_t)channel_plane; + _mm256_storeu_ps(output0, accumulator0); + if (valid_outputs > 1u) { + _mm256_storeu_ps(output1, accumulator1); + } + if (valid_outputs > 2u) { + _mm256_storeu_ps(output2, accumulator2); + } + if (valid_outputs > 3u) { + _mm256_storeu_ps(output3, accumulator3); + } + } + for (; spatial < channel_plane; ++spatial) { + float accumulator0 = bias == NULL ? 0.0f : bias[output_base]; + float accumulator1 = + bias == NULL || valid_outputs <= 1u ? 0.0f : bias[output_base + 1u]; + float accumulator2 = + bias == NULL || valid_outputs <= 2u ? 0.0f : bias[output_base + 2u]; + float accumulator3 = + bias == NULL || valid_outputs <= 3u ? 0.0f : bias[output_base + 3u]; + const float* input_ptr = batch_input + (size_t)spatial; + const float* packed_ptr = + packed_weights + + (size_t)((uint64_t)output_block * input_channels * + LW_PACKED_CONV1X1_OUTPUT_TILE); + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + float input_value = *input_ptr; + accumulator0 += input_value * packed_ptr[0]; + accumulator1 += input_value * packed_ptr[1]; + accumulator2 += input_value * packed_ptr[2]; + accumulator3 += input_value * packed_ptr[3]; + input_ptr += (size_t)channel_plane; + packed_ptr += LW_PACKED_CONV1X1_OUTPUT_TILE; + } + float* output0 = + batch_output + (size_t)((uint64_t)output_base * channel_plane + spatial); + float* output1 = output0 + (size_t)channel_plane; + float* output2 = output1 + (size_t)channel_plane; + float* output3 = output2 + (size_t)channel_plane; + *output0 = accumulator0; + if (valid_outputs > 1u) + *output1 = accumulator1; + if (valid_outputs > 2u) + *output2 = accumulator2; + if (valid_outputs > 3u) + *output3 = accumulator3; + } + } + } +#else + lw_scalar_packed_conv1x1_f32(input, packed_weights, bias, output, input_dimensions, + output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_fma_packed_conv1x1_8x8.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_fma_packed_conv1x1_8x8.c new file mode 100644 index 00000000..9f1a821f --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_fma_packed_conv1x1_8x8.c @@ -0,0 +1,150 @@ +#include "simd_kernels.h" + +#include "packed_conv_internal.h" + +#include + +#if defined(_M_X64) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_FMA_PACKED_CONV1X1_8X8 1 +#else +# define LW_COMPILES_AVX2_FMA_PACKED_CONV1X1_8X8 0 +#endif + +#if LW_COMPILES_AVX2_FMA_PACKED_CONV1X1_8X8 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,fma"))) +#endif +void lw_avx2_fma_packed_conv1x1_8x8_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_FMA_PACKED_CONV1X1_8X8 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint64_t channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + const uint32_t batch_count = (uint32_t)input_dimensions[0]; + uint32_t batch; + + /* The candidate owns complete pairs of packed 4-output blocks. Keep all + * other shapes on the validated 4-output FMA kernel. */ + if (input == NULL || packed_weights == NULL || output == NULL || + output_channels < 8u || (output_channels & 7u) != 0u || channel_plane < 8u) { + lw_avx2_fma_packed_conv1x1_f32(input, packed_weights, bias, output, + input_dimensions, output_dimensions); + return; + } + + for (batch = 0u; batch < batch_count; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_base; + for (output_base = 0u; output_base < output_channels; output_base += 8u) { + const uint32_t packed_block = output_base / LW_PACKED_CONV1X1_OUTPUT_TILE; + const float* packed0 = + packed_weights + + (size_t)((uint64_t)packed_block * input_channels * + LW_PACKED_CONV1X1_OUTPUT_TILE); + const float* packed1 = + packed0 + (size_t)((uint64_t)input_channels * + LW_PACKED_CONV1X1_OUTPUT_TILE); + uint64_t spatial = 0u; + for (; spatial + 8u <= channel_plane; spatial += 8u) { + __m256 accumulator0 = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base]); + __m256 accumulator1 = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 1u]); + __m256 accumulator2 = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 2u]); + __m256 accumulator3 = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 3u]); + __m256 accumulator4 = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 4u]); + __m256 accumulator5 = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 5u]); + __m256 accumulator6 = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 6u]); + __m256 accumulator7 = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 7u]); + const float* input_ptr = batch_input + (size_t)spatial; + const float* packed_ptr0 = packed0; + const float* packed_ptr1 = packed1; + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const __m256 input_values = _mm256_loadu_ps(input_ptr); + accumulator0 = _mm256_fmadd_ps(input_values, _mm256_set1_ps(packed_ptr0[0]), accumulator0); + accumulator1 = _mm256_fmadd_ps(input_values, _mm256_set1_ps(packed_ptr0[1]), accumulator1); + accumulator2 = _mm256_fmadd_ps(input_values, _mm256_set1_ps(packed_ptr0[2]), accumulator2); + accumulator3 = _mm256_fmadd_ps(input_values, _mm256_set1_ps(packed_ptr0[3]), accumulator3); + accumulator4 = _mm256_fmadd_ps(input_values, _mm256_set1_ps(packed_ptr1[0]), accumulator4); + accumulator5 = _mm256_fmadd_ps(input_values, _mm256_set1_ps(packed_ptr1[1]), accumulator5); + accumulator6 = _mm256_fmadd_ps(input_values, _mm256_set1_ps(packed_ptr1[2]), accumulator6); + accumulator7 = _mm256_fmadd_ps(input_values, _mm256_set1_ps(packed_ptr1[3]), accumulator7); + input_ptr += (size_t)channel_plane; + packed_ptr0 += LW_PACKED_CONV1X1_OUTPUT_TILE; + packed_ptr1 += LW_PACKED_CONV1X1_OUTPUT_TILE; + } + float* output0 = + batch_output + (size_t)((uint64_t)output_base * channel_plane + spatial); + float* output1 = output0 + (size_t)channel_plane; + float* output2 = output1 + (size_t)channel_plane; + float* output3 = output2 + (size_t)channel_plane; + float* output4 = output3 + (size_t)channel_plane; + float* output5 = output4 + (size_t)channel_plane; + float* output6 = output5 + (size_t)channel_plane; + float* output7 = output6 + (size_t)channel_plane; + _mm256_storeu_ps(output0, accumulator0); + _mm256_storeu_ps(output1, accumulator1); + _mm256_storeu_ps(output2, accumulator2); + _mm256_storeu_ps(output3, accumulator3); + _mm256_storeu_ps(output4, accumulator4); + _mm256_storeu_ps(output5, accumulator5); + _mm256_storeu_ps(output6, accumulator6); + _mm256_storeu_ps(output7, accumulator7); + } + + for (; spatial < channel_plane; ++spatial) { + float accumulator0 = bias == NULL ? 0.0f : bias[output_base]; + float accumulator1 = bias == NULL ? 0.0f : bias[output_base + 1u]; + float accumulator2 = bias == NULL ? 0.0f : bias[output_base + 2u]; + float accumulator3 = bias == NULL ? 0.0f : bias[output_base + 3u]; + float accumulator4 = bias == NULL ? 0.0f : bias[output_base + 4u]; + float accumulator5 = bias == NULL ? 0.0f : bias[output_base + 5u]; + float accumulator6 = bias == NULL ? 0.0f : bias[output_base + 6u]; + float accumulator7 = bias == NULL ? 0.0f : bias[output_base + 7u]; + const float* input_ptr = batch_input + (size_t)spatial; + const float* packed_ptr0 = packed0; + const float* packed_ptr1 = packed1; + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float input_value = *input_ptr; + accumulator0 += input_value * packed_ptr0[0]; + accumulator1 += input_value * packed_ptr0[1]; + accumulator2 += input_value * packed_ptr0[2]; + accumulator3 += input_value * packed_ptr0[3]; + accumulator4 += input_value * packed_ptr1[0]; + accumulator5 += input_value * packed_ptr1[1]; + accumulator6 += input_value * packed_ptr1[2]; + accumulator7 += input_value * packed_ptr1[3]; + input_ptr += (size_t)channel_plane; + packed_ptr0 += LW_PACKED_CONV1X1_OUTPUT_TILE; + packed_ptr1 += LW_PACKED_CONV1X1_OUTPUT_TILE; + } + float* output0 = + batch_output + (size_t)((uint64_t)output_base * channel_plane + spatial); + float* output1 = output0 + (size_t)channel_plane; + float* output2 = output1 + (size_t)channel_plane; + float* output3 = output2 + (size_t)channel_plane; + float* output4 = output3 + (size_t)channel_plane; + float* output5 = output4 + (size_t)channel_plane; + float* output6 = output5 + (size_t)channel_plane; + float* output7 = output6 + (size_t)channel_plane; + *output0 = accumulator0; + *output1 = accumulator1; + *output2 = accumulator2; + *output3 = accumulator3; + *output4 = accumulator4; + *output5 = accumulator5; + *output6 = accumulator6; + *output7 = accumulator7; + } + } + } +#else + lw_avx2_fma_packed_conv1x1_f32(input, packed_weights, bias, output, + input_dimensions, output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_fma_packed_matmul.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_fma_packed_matmul.c new file mode 100644 index 00000000..69f4a46e --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_fma_packed_matmul.c @@ -0,0 +1,321 @@ +#include "simd_kernels.h" + +#include "packed_matmul_internal.h" + +#include +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_FMA_MATMUL 1 +#else +# define LW_COMPILES_AVX2_FMA_MATMUL 0 +#endif + +#if LW_COMPILES_AVX2_FMA_MATMUL && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,fma"))) +#endif +void lw_avx2_fma_packed_matmul_bias_argmax_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + uint32_t* best_indices, uint32_t batch_count, uint32_t rows, + uint32_t inner_dimension, uint32_t columns) { +#if LW_COMPILES_AVX2_FMA_MATMUL + const uint32_t column_panels = + (columns + LW_PACKED_MATMUL_COLUMN_TILE - 1u) / LW_PACKED_MATMUL_COLUMN_TILE; + uint32_t batch; + for (batch = 0u; batch < batch_count; ++batch) { + uint32_t row; + for (row = 0u; row + 4u <= rows; row += 4u) { + uint32_t column_panel; + for (column_panel = 0u; column_panel < column_panels; ++column_panel) { + __m256 accumulators[8] = { + _mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps(), + _mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps(), + _mm256_setzero_ps(), _mm256_setzero_ps()}; + uint32_t inner; + for (inner = 0u; inner < inner_dimension; ++inner) { + const float* packed = + packed_weights + + (size_t)(((uint64_t)column_panel * inner_dimension + inner) * + LW_PACKED_MATMUL_COLUMN_TILE); + const uint64_t input_base = + ((uint64_t)batch * rows + row) * inner_dimension + inner; + const __m256 weight_low = _mm256_loadu_ps(packed); + const __m256 weight_high = _mm256_loadu_ps(packed + 8u); + __m256 input_value = _mm256_set1_ps(input[(size_t)input_base]); + accumulators[0] = _mm256_fmadd_ps(input_value, weight_low, accumulators[0]); + accumulators[1] = _mm256_fmadd_ps(input_value, weight_high, accumulators[1]); + input_value = _mm256_set1_ps(input[(size_t)(input_base + inner_dimension)]); + accumulators[2] = _mm256_fmadd_ps(input_value, weight_low, accumulators[2]); + accumulators[3] = _mm256_fmadd_ps(input_value, weight_high, accumulators[3]); + input_value = + _mm256_set1_ps(input[(size_t)(input_base + 2u * inner_dimension)]); + accumulators[4] = _mm256_fmadd_ps(input_value, weight_low, accumulators[4]); + accumulators[5] = _mm256_fmadd_ps(input_value, weight_high, accumulators[5]); + input_value = + _mm256_set1_ps(input[(size_t)(input_base + 3u * inner_dimension)]); + accumulators[6] = _mm256_fmadd_ps(input_value, weight_low, accumulators[6]); + accumulators[7] = _mm256_fmadd_ps(input_value, weight_high, accumulators[7]); + } + { + const uint32_t column_base = column_panel * LW_PACKED_MATMUL_COLUMN_TILE; + const uint32_t valid_columns = + columns - column_base < LW_PACKED_MATMUL_COLUMN_TILE + ? columns - column_base + : LW_PACKED_MATMUL_COLUMN_TILE; + uint32_t current_row; + for (current_row = 0u; current_row < 4u; ++current_row) { + const uint64_t result_row = (uint64_t)batch * rows + row + current_row; + float* destination = + output + (size_t)(result_row * columns + column_base); + float values[LW_PACKED_MATMUL_COLUMN_TILE]; + uint32_t best_index = column_base == 0u + ? 0u + : best_indices[(size_t)result_row]; + float best_value = column_base == 0u + ? 0.0f + : output[(size_t)(result_row * columns + + best_index)]; + uint32_t lane; + _mm256_storeu_ps(values, accumulators[current_row * 2u]); + _mm256_storeu_ps(values + 8u, accumulators[current_row * 2u + 1u]); + for (lane = 0u; lane < valid_columns; ++lane) { + const float value = values[lane] + bias[column_base + lane]; + destination[lane] = value; + if ((column_base != 0u || lane != 0u) && value > best_value) { + best_value = value; + best_index = column_base + lane; + } else if (column_base == 0u && lane == 0u) { + best_value = value; + } + } + best_indices[(size_t)result_row] = best_index; + } + } + } + } + } +#else + (void)input; + (void)packed_weights; + (void)bias; + (void)output; + (void)best_indices; + (void)batch_count; + (void)rows; + (void)inner_dimension; + (void)columns; +#endif +} + +#if LW_COMPILES_AVX2_FMA_MATMUL +/* Strict-greater vector running max: replaces the value and its index only + * when the new value is strictly greater, which keeps the lowest column index + * on ties (same semantics as the stored-logits kernel). */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,fma"))) +#endif +static void update_max256(__m256* maxima, __m256* maxima_indices, __m256 values, + uint32_t column_base, __m256 lane_offsets) { + const __m256 candidates = + _mm256_add_ps(_mm256_set1_ps((float)column_base), lane_offsets); + const __m256 mask = _mm256_cmp_ps(values, *maxima, _CMP_GT_OQ); + *maxima = _mm256_max_ps(*maxima, values); + *maxima_indices = _mm256_blendv_ps(*maxima_indices, candidates, mask); +} + +/* Scalar horizontal reduce of one 8-lane max/index pair; strict-greater + * keeps the lowest lane index on ties. */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,fma"))) +#endif +static void reduce_max8(__m256 values, __m256 indices, float* best_value, + uint32_t* best_index) { + float lane_values[8]; + float lane_indices[8]; + uint32_t lane; + _mm256_storeu_ps(lane_values, values); + _mm256_storeu_ps(lane_indices, indices); + for (lane = 0u; lane < 8u; ++lane) { + if (lane_values[lane] > *best_value) { + *best_value = lane_values[lane]; + *best_index = (uint32_t)lane_indices[lane]; + } + } +} +#endif + +#if LW_COMPILES_AVX2_FMA_MATMUL && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,fma"))) +#endif +void lw_avx2_fma_packed_matmul_argmax_scores_f32( + const float* input, const float* packed_weights, const float* bias, + uint32_t* best_indices, float* scores, uint32_t batch_count, uint32_t rows, + uint32_t inner_dimension, uint32_t columns) { +#if LW_COMPILES_AVX2_FMA_MATMUL + /* Panel-outer loop order with row chunks: the packed weights for one + * column panel stay cache-resident while all rows of the chunk are + * processed, so the weight matrix streams from memory once per chunk + * instead of once per 4-row block. Column panels are disjoint output + * column blocks and every output element accumulates over the inner + * dimension in the same order as the row-outer form, and each row's + * running maximum still visits panels in ascending order, so results + * are bit-identical. */ +#define LW_CTC_HEAD_ROW_CHUNK 128u + const uint32_t column_panels = + (columns + LW_PACKED_MATMUL_COLUMN_TILE - 1u) / LW_PACKED_MATMUL_COLUMN_TILE; + const uint32_t aligned_rows = rows & ~3u; + const int has_partial_panel = (columns % LW_PACKED_MATMUL_COLUMN_TILE) != 0u; + const __m256 lane_offsets = _mm256_set_ps(7.0f, 6.0f, 5.0f, 4.0f, + 3.0f, 2.0f, 1.0f, 0.0f); + uint32_t batch; + for (batch = 0u; batch < batch_count; ++batch) { + uint32_t chunk_begin; + for (chunk_begin = 0u; chunk_begin < aligned_rows; + chunk_begin += LW_CTC_HEAD_ROW_CHUNK) { + uint32_t chunk_rows = aligned_rows - chunk_begin; + __m256 maxima[LW_CTC_HEAD_ROW_CHUNK * 2u]; + __m256 maxima_indices[LW_CTC_HEAD_ROW_CHUNK * 2u]; + float partial_values[LW_CTC_HEAD_ROW_CHUNK][LW_PACKED_MATMUL_COLUMN_TILE]; + uint32_t chunk_row; + uint32_t column_panel; + if (chunk_rows > LW_CTC_HEAD_ROW_CHUNK) { + chunk_rows = LW_CTC_HEAD_ROW_CHUNK; + } + for (chunk_row = 0u; chunk_row < chunk_rows; ++chunk_row) { + const __m256 negative_infinity = _mm256_set1_ps(-(float)INFINITY); + maxima[chunk_row * 2u] = negative_infinity; + maxima[chunk_row * 2u + 1u] = negative_infinity; + maxima_indices[chunk_row * 2u] = _mm256_setzero_ps(); + maxima_indices[chunk_row * 2u + 1u] = _mm256_setzero_ps(); + } + for (column_panel = 0u; column_panel < column_panels; ++column_panel) { + const uint32_t column_base = + column_panel * LW_PACKED_MATMUL_COLUMN_TILE; + const uint32_t valid_columns = + columns - column_base < LW_PACKED_MATMUL_COLUMN_TILE + ? columns - column_base + : LW_PACKED_MATMUL_COLUMN_TILE; + const int full_panel = valid_columns == LW_PACKED_MATMUL_COLUMN_TILE; + for (chunk_row = 0u; chunk_row < chunk_rows; chunk_row += 4u) { + const uint32_t row = chunk_begin + chunk_row; + __m256 accumulators[8] = { + _mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps(), + _mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps(), + _mm256_setzero_ps(), _mm256_setzero_ps()}; + uint32_t inner; + uint32_t current_row; + for (inner = 0u; inner < inner_dimension; ++inner) { + const float* packed = + packed_weights + + (size_t)(((uint64_t)column_panel * inner_dimension + inner) * + LW_PACKED_MATMUL_COLUMN_TILE); + const uint64_t input_base = + ((uint64_t)batch * rows + row) * inner_dimension + inner; + const __m256 weight_low = _mm256_loadu_ps(packed); + const __m256 weight_high = _mm256_loadu_ps(packed + 8u); + __m256 input_value = _mm256_set1_ps(input[(size_t)input_base]); + accumulators[0] = _mm256_fmadd_ps(input_value, weight_low, accumulators[0]); + accumulators[1] = _mm256_fmadd_ps(input_value, weight_high, accumulators[1]); + input_value = _mm256_set1_ps(input[(size_t)(input_base + inner_dimension)]); + accumulators[2] = _mm256_fmadd_ps(input_value, weight_low, accumulators[2]); + accumulators[3] = _mm256_fmadd_ps(input_value, weight_high, accumulators[3]); + input_value = + _mm256_set1_ps(input[(size_t)(input_base + 2u * inner_dimension)]); + accumulators[4] = _mm256_fmadd_ps(input_value, weight_low, accumulators[4]); + accumulators[5] = _mm256_fmadd_ps(input_value, weight_high, accumulators[5]); + input_value = + _mm256_set1_ps(input[(size_t)(input_base + 3u * inner_dimension)]); + accumulators[6] = _mm256_fmadd_ps(input_value, weight_low, accumulators[6]); + accumulators[7] = _mm256_fmadd_ps(input_value, weight_high, accumulators[7]); + } + if (bias != NULL) { + __m256 bias_low; + __m256 bias_high; + if (full_panel) { + bias_low = _mm256_loadu_ps(bias + column_base); + bias_high = _mm256_loadu_ps(bias + column_base + 8u); + } else { + float bias_values[LW_PACKED_MATMUL_COLUMN_TILE] = {0.0f}; + uint32_t lane; + for (lane = 0u; lane < valid_columns; ++lane) { + bias_values[lane] = bias[column_base + lane]; + } + bias_low = _mm256_loadu_ps(bias_values); + bias_high = _mm256_loadu_ps(bias_values + 8u); + } + for (current_row = 0u; current_row < 4u; ++current_row) { + accumulators[current_row * 2u] = + _mm256_add_ps(accumulators[current_row * 2u], bias_low); + accumulators[current_row * 2u + 1u] = + _mm256_add_ps(accumulators[current_row * 2u + 1u], bias_high); + } + } + if (full_panel) { + for (current_row = 0u; current_row < 4u; ++current_row) { + uint32_t state_row = chunk_row + current_row; + update_max256(&maxima[state_row * 2u], + &maxima_indices[state_row * 2u], + accumulators[current_row * 2u], column_base, + lane_offsets); + update_max256(&maxima[state_row * 2u + 1u], + &maxima_indices[state_row * 2u + 1u], + accumulators[current_row * 2u + 1u], + column_base + 8u, lane_offsets); + } + } else { + /* Partial final panel: keep the raw values per row; + * the scalar merge runs after the vector reduction, + * exactly like the row-outer form. */ + for (current_row = 0u; current_row < 4u; ++current_row) { + _mm256_storeu_ps(partial_values[chunk_row + current_row], + accumulators[current_row * 2u]); + _mm256_storeu_ps(partial_values[chunk_row + current_row] + 8u, + accumulators[current_row * 2u + 1u]); + } + } + } + } + for (chunk_row = 0u; chunk_row < chunk_rows; ++chunk_row) { + const uint64_t result_row = + (uint64_t)batch * rows + chunk_begin + chunk_row; + float best_value = -(float)INFINITY; + uint32_t best_index = 0u; + reduce_max8(maxima[chunk_row * 2u], maxima_indices[chunk_row * 2u], + &best_value, &best_index); + reduce_max8(maxima[chunk_row * 2u + 1u], + maxima_indices[chunk_row * 2u + 1u], + &best_value, &best_index); + if (has_partial_panel) { + const uint32_t column_base = + (column_panels - 1u) * LW_PACKED_MATMUL_COLUMN_TILE; + const uint32_t valid_columns = columns - column_base; + uint32_t lane; + for (lane = 0u; lane < valid_columns; ++lane) { + const float value = partial_values[chunk_row][lane]; + if ((column_base != 0u || lane != 0u) && + value > best_value) { + best_value = value; + best_index = column_base + lane; + } else if (column_base == 0u && lane == 0u) { + best_value = value; + } + } + } + scores[(size_t)result_row] = best_value; + best_indices[(size_t)result_row] = best_index; + } + } + } +#else + (void)input; + (void)packed_weights; + (void)bias; + (void)best_indices; + (void)scores; + (void)batch_count; + (void)rows; + (void)inner_dimension; + (void)columns; +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_matmul.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_matmul.c new file mode 100644 index 00000000..17e79f1e --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_matmul.c @@ -0,0 +1,350 @@ +#include "simd_kernels.h" + +#if defined(LW_EXPERIMENTAL_AVX2_FMA_MATMUL_DISPATCH) +# include "cpu_features.h" +#endif + +#include "packed_matmul_internal.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2 1 +#else +# define LW_COMPILES_AVX2 0 +#endif + +#if LW_COMPILES_AVX2 +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static void lw_avx2_matmul_rows4_tiled_f32( + const float* input, + const float* weights, + float* output, + uint32_t batch_count, + uint32_t rows, + uint32_t inner_dimension, + uint32_t columns) { + uint32_t batch; + for (batch = 0u; batch < batch_count; ++batch) { + uint32_t row; + for (row = 0u; row + 4u <= rows; row += 4u) { + uint32_t column; + for (column = 0u; column < columns; column += 8u) { + uint32_t width = columns - column < 8u ? columns - column : 8u; + __m256 accumulators[4] = { + _mm256_setzero_ps(), _mm256_setzero_ps(), + _mm256_setzero_ps(), _mm256_setzero_ps()}; + uint32_t inner; + for (inner = 0u; inner < inner_dimension; ++inner) { + const uint64_t weight_base = (uint64_t)inner * columns + column; + const uint64_t input_base = ((uint64_t)batch * rows + row) * inner_dimension + inner; + __m256 weight_values; + if (width == 8u) { + weight_values = _mm256_loadu_ps(weights + (size_t)weight_base); + } else { + /* The final tile may contain fewer than eight columns. + * Copy only valid weights before loading the vector so + * this fast path never reads past the matrix. */ + float weight_tail[8] = {0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f}; + for (uint32_t lane = 0u; lane < width; ++lane) { + weight_tail[lane] = weights[(size_t)weight_base + lane]; + } + weight_values = _mm256_loadu_ps(weight_tail); + } + uint32_t current_row; + for (current_row = 0u; current_row < 4u; ++current_row) { + __m256 input_value = _mm256_set1_ps( + input[(size_t)(input_base + (uint64_t)current_row * inner_dimension)]); + accumulators[current_row] = _mm256_add_ps( + accumulators[current_row], _mm256_mul_ps(input_value, weight_values)); + } + } + { + uint32_t current_row; + for (current_row = 0u; current_row < 4u; ++current_row) { + float* destination = output + (size_t)(((uint64_t)batch * rows + row + current_row) * columns + column); + if (width == 8u) { + _mm256_storeu_ps(destination, accumulators[current_row]); + } else { + float values[8]; + _mm256_storeu_ps(values, accumulators[current_row]); + for (uint32_t lane = 0u; lane < width; ++lane) { + destination[lane] = values[lane]; + } + } + } + } + } + } + } +} +#endif + +#if LW_COMPILES_AVX2 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_matmul_shared_f32( + const float* input, + const float* weights, + float* output, + uint32_t batch_count, + uint32_t rows, + uint32_t inner_dimension, + uint32_t columns) { +#if LW_COMPILES_AVX2 + if (rows >= 4u && (rows % 4u) == 0u && inner_dimension >= 64u && columns >= 1024u) { + lw_avx2_matmul_rows4_tiled_f32(input, weights, output, batch_count, rows, + inner_dimension, columns); + return; + } +#endif + uint32_t batch; + uint32_t row; + for (batch = 0u; batch < batch_count; ++batch) { + for (row = 0u; row < rows;) { + uint32_t row_end = rows - row < 4u ? rows : row + 4u; + uint32_t inner; + uint32_t current_row; + for (current_row = row; current_row < row_end; ++current_row) { + uint64_t output_base = ((uint64_t)batch * rows + current_row) * columns; + uint32_t column; + for (column = 0u; column < columns; ++column) { + output[(size_t)(output_base + column)] = 0.0f; + } + } + for (inner = 0u; inner < inner_dimension; ++inner) { + uint64_t weight_base = (uint64_t)inner * columns; + for (current_row = row; current_row < row_end; ++current_row) { + uint64_t input_base = ((uint64_t)batch * rows + current_row) * inner_dimension; + uint64_t output_base = ((uint64_t)batch * rows + current_row) * columns; + float input_value = input[(size_t)(input_base + inner)]; + uint32_t column = 0u; +#if LW_COMPILES_AVX2 + __m256 input_values = _mm256_set1_ps(input_value); + for (; column + 8u <= columns; column += 8u) { + __m256 output_values = + _mm256_loadu_ps(output + (size_t)(output_base + column)); + __m256 weight_values = + _mm256_loadu_ps(weights + (size_t)(weight_base + column)); + output_values = _mm256_add_ps(output_values, + _mm256_mul_ps(input_values, weight_values)); + _mm256_storeu_ps(output + (size_t)(output_base + column), output_values); + } +#endif + for (; column < columns; ++column) { + output[(size_t)(output_base + column)] += + input_value * weights[(size_t)(weight_base + column)]; + } + } + } + row = row_end; + } + } +} + +#if LW_COMPILES_AVX2 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +static void lw_avx2_packed_matmul_impl_f32( + const float* input, + const float* packed_weights, + const float* bias, + float* output, + uint32_t* best_indices, + uint32_t batch_count, + uint32_t rows, + uint32_t inner_dimension, + uint32_t columns) { +#if LW_COMPILES_AVX2 && (defined(_M_X64) || defined(__x86_64__)) + uint32_t batch; + const uint32_t column_panels = + (columns + LW_PACKED_MATMUL_COLUMN_TILE - 1u) / LW_PACKED_MATMUL_COLUMN_TILE; + if (rows < 4u || rows % 4u != 0u) { + lw_scalar_packed_matmul_shared_f32(input, packed_weights, output, batch_count, rows, + inner_dimension, columns); + if (bias != NULL && best_indices != NULL) { + uint32_t fallback_batch; + for (fallback_batch = 0u; fallback_batch < batch_count; ++fallback_batch) { + uint32_t row; + for (row = 0u; row < rows; ++row) { + const uint64_t output_base = + ((uint64_t)fallback_batch * rows + row) * columns; + uint32_t best_index = 0u; + uint32_t column; + for (column = 0u; column < columns; ++column) { + output[(size_t)(output_base + column)] += bias[column]; + if (column != 0u && output[(size_t)(output_base + column)] > + output[(size_t)(output_base + best_index)]) { + best_index = column; + } + } + best_indices[(size_t)((uint64_t)fallback_batch * rows + row)] = best_index; + } + } + } + return; + } + for (batch = 0u; batch < batch_count; ++batch) { + uint32_t row; + for (row = 0u; row + 4u <= rows; row += 4u) { + uint32_t column_panel; + for (column_panel = 0u; column_panel < column_panels; ++column_panel) { + __m256 accumulator0_low = _mm256_setzero_ps(); + __m256 accumulator0_high = _mm256_setzero_ps(); + __m256 accumulator1_low = _mm256_setzero_ps(); + __m256 accumulator1_high = _mm256_setzero_ps(); + __m256 accumulator2_low = _mm256_setzero_ps(); + __m256 accumulator2_high = _mm256_setzero_ps(); + __m256 accumulator3_low = _mm256_setzero_ps(); + __m256 accumulator3_high = _mm256_setzero_ps(); + uint32_t inner; + for (inner = 0u; inner < inner_dimension; ++inner) { + const float* packed = + packed_weights + + (size_t)(((uint64_t)column_panel * inner_dimension + inner) * + LW_PACKED_MATMUL_COLUMN_TILE); + const uint64_t input_base = + ((uint64_t)batch * rows + row) * inner_dimension + inner; + const __m256 weight_low = _mm256_loadu_ps(packed); + const __m256 weight_high = _mm256_loadu_ps(packed + 8u); + __m256 input_value = _mm256_set1_ps(input[(size_t)input_base]); + accumulator0_low = _mm256_add_ps( + accumulator0_low, _mm256_mul_ps(input_value, weight_low)); + accumulator0_high = _mm256_add_ps( + accumulator0_high, _mm256_mul_ps(input_value, weight_high)); + input_value = + _mm256_set1_ps(input[(size_t)(input_base + inner_dimension)]); + accumulator1_low = _mm256_add_ps( + accumulator1_low, _mm256_mul_ps(input_value, weight_low)); + accumulator1_high = _mm256_add_ps( + accumulator1_high, _mm256_mul_ps(input_value, weight_high)); + input_value = + _mm256_set1_ps(input[(size_t)(input_base + 2u * inner_dimension)]); + accumulator2_low = _mm256_add_ps( + accumulator2_low, _mm256_mul_ps(input_value, weight_low)); + accumulator2_high = _mm256_add_ps( + accumulator2_high, _mm256_mul_ps(input_value, weight_high)); + input_value = + _mm256_set1_ps(input[(size_t)(input_base + 3u * inner_dimension)]); + accumulator3_low = _mm256_add_ps( + accumulator3_low, _mm256_mul_ps(input_value, weight_low)); + accumulator3_high = _mm256_add_ps( + accumulator3_high, _mm256_mul_ps(input_value, weight_high)); + } + { + const uint32_t column_base = column_panel * LW_PACKED_MATMUL_COLUMN_TILE; + const uint32_t valid_columns = + columns - column_base < LW_PACKED_MATMUL_COLUMN_TILE + ? columns - column_base + : LW_PACKED_MATMUL_COLUMN_TILE; + __m256 accumulators[8] = { + accumulator0_low, accumulator0_high, accumulator1_low, accumulator1_high, + accumulator2_low, accumulator2_high, accumulator3_low, accumulator3_high}; + uint32_t current_row; + for (current_row = 0u; current_row < 4u; ++current_row) { + const uint64_t result_row = (uint64_t)batch * rows + row + current_row; + float* destination = + output + (size_t)(result_row * columns + column_base); + if (bias == NULL || best_indices == NULL) { + if (valid_columns == LW_PACKED_MATMUL_COLUMN_TILE) { + _mm256_storeu_ps(destination, accumulators[current_row * 2u]); + _mm256_storeu_ps(destination + 8u, + accumulators[current_row * 2u + 1u]); + } else { + float values[LW_PACKED_MATMUL_COLUMN_TILE]; + uint32_t lane; + _mm256_storeu_ps(values, accumulators[current_row * 2u]); + _mm256_storeu_ps(values + 8u, + accumulators[current_row * 2u + 1u]); + for (lane = 0u; lane < valid_columns; ++lane) { + destination[lane] = values[lane]; + } + } + } else { + float values[LW_PACKED_MATMUL_COLUMN_TILE]; + uint32_t best_index = column_base == 0u + ? 0u + : best_indices[(size_t)result_row]; + float best_value = column_base == 0u + ? 0.0f + : output[(size_t)(result_row * columns + + best_index)]; + uint32_t lane; + _mm256_storeu_ps(values, accumulators[current_row * 2u]); + _mm256_storeu_ps(values + 8u, accumulators[current_row * 2u + 1u]); + for (lane = 0u; lane < valid_columns; ++lane) { + float value = values[lane] + bias[column_base + lane]; + destination[lane] = value; + if ((column_base != 0u || lane != 0u) && value > best_value) { + best_value = value; + best_index = column_base + lane; + } else if (column_base == 0u && lane == 0u) { + best_value = value; + } + } + best_indices[(size_t)result_row] = best_index; + } + } + } + } + } + } +#else + lw_scalar_packed_matmul_shared_f32(input, packed_weights, output, batch_count, rows, + inner_dimension, columns); + if (bias != NULL && best_indices != NULL) { + uint32_t batch; + for (batch = 0u; batch < batch_count; ++batch) { + uint32_t row; + for (row = 0u; row < rows; ++row) { + const uint64_t output_base = ((uint64_t)batch * rows + row) * columns; + uint32_t best_index = 0u; + uint32_t column; + for (column = 0u; column < columns; ++column) { + output[(size_t)(output_base + column)] += bias[column]; + if (column != 0u && output[(size_t)(output_base + column)] > + output[(size_t)(output_base + best_index)]) { + best_index = column; + } + } + best_indices[(size_t)((uint64_t)batch * rows + row)] = best_index; + } + } + } +#endif +} + +#if LW_COMPILES_AVX2 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_packed_matmul_shared_f32( + const float* input, const float* packed_weights, float* output, uint32_t batch_count, + uint32_t rows, uint32_t inner_dimension, uint32_t columns) { + lw_avx2_packed_matmul_impl_f32(input, packed_weights, NULL, output, NULL, batch_count, rows, + inner_dimension, columns); +} + +#if LW_COMPILES_AVX2 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_packed_matmul_bias_argmax_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + uint32_t* best_indices, uint32_t batch_count, uint32_t rows, + uint32_t inner_dimension, uint32_t columns) { +#if defined(LW_EXPERIMENTAL_AVX2_FMA_MATMUL_DISPATCH) + const lw_cpu_capabilities capabilities = lw_get_cpu_capabilities(); + if (capabilities.has_avx2_fma && batch_count == 1u && rows == 40u && + inner_dimension == 80u && columns == 6906u) { + lw_avx2_fma_packed_matmul_bias_argmax_f32( + input, packed_weights, bias, output, best_indices, batch_count, rows, + inner_dimension, columns); + return; + } +#endif + lw_avx2_packed_matmul_impl_f32(input, packed_weights, bias, output, best_indices, + batch_count, rows, inner_dimension, columns); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_convtranspose.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_convtranspose.c new file mode 100644 index 00000000..539eb4ce --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_convtranspose.c @@ -0,0 +1,309 @@ +#include "nhwc_internal.h" + +#include +#include + +#if defined(_M_X64) || defined(__x86_64__) || defined(_M_IX86) || defined(__i386__) +#include +#define LW_CT_X86 1 +#else +#define LW_CT_X86 0 +#endif + +#if LW_CT_X86 && (defined(__GNUC__) || defined(__clang__)) +#define LW_CT_TARGET __attribute__((target("avx2,fma"))) +#else +#define LW_CT_TARGET +#endif + +#define LW_CT_TILE 6u +#define LW_CT_TAPS 4u + +int lw_nhwc_convtranspose_packed_weight_count(uint32_t input_channels, + uint32_t output_channels, + uint64_t* element_count) { + uint64_t blocks; + uint64_t count; + if (input_channels == 0u || output_channels == 0u || + (output_channels & 7u) != 0u || element_count == NULL) { + return 0; + } + blocks = (output_channels + LW_NHWC_OC_BLOCK - 1u) / LW_NHWC_OC_BLOCK; + count = blocks; + if (count > UINT64_MAX / input_channels) return 0; + count *= input_channels; + if (count > UINT64_MAX / LW_CT_TAPS) return 0; + count *= LW_CT_TAPS; + if (count > UINT64_MAX / LW_NHWC_OC_BLOCK) return 0; + count *= LW_NHWC_OC_BLOCK; + if (count > (uint64_t)(SIZE_MAX / sizeof(float))) return 0; + *element_count = count; + return 1; +} + +void lw_pack_nhwc_convtranspose2x2_f32(const float* weights, uint32_t input_channels, + uint32_t output_channels, float* packed_weights) { + /* ONNX ConvTranspose weights [ic, oc, 2, 2] -> [tap][oc/16][ic][16] with + * tap = dy*2+dx. */ + uint32_t blocks = (output_channels + LW_NHWC_OC_BLOCK - 1u) / LW_NHWC_OC_BLOCK; + uint32_t tap; + for (tap = 0u; tap < LW_CT_TAPS; ++tap) { + uint32_t block; + for (block = 0u; block < blocks; ++block) { + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + uint32_t lane; + for (lane = 0u; lane < LW_NHWC_OC_BLOCK; ++lane) { + uint32_t output_channel = block * LW_NHWC_OC_BLOCK + lane; + size_t packed_index = + ((((size_t)tap * blocks + block) * input_channels + input_channel) * + LW_NHWC_OC_BLOCK) + lane; + size_t weight_index = ((size_t)input_channel * output_channels + + output_channel) * LW_CT_TAPS + tap; + packed_weights[packed_index] = output_channel < output_channels + ? weights[weight_index] : 0.0f; + } + } + } + } +} + +#if LW_CT_X86 +LW_CT_TARGET +static void convtranspose_tile_16(const float* input_row, const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output_row0, float* output_row1, + const lw_nhwc_convtranspose_desc* desc, + uint32_t oc_blocks, uint32_t input_x, + uint32_t pixels) { + uint32_t block; + for (block = 0u; block < oc_blocks; ++block) { + uint32_t block_channels = desc->output_channels - block * LW_NHWC_OC_BLOCK; + uint32_t tap; + __m256 bias_lo = _mm256_setzero_ps(); + __m256 bias_hi = _mm256_setzero_ps(); + if (block_channels > LW_NHWC_OC_BLOCK) block_channels = LW_NHWC_OC_BLOCK; + if (epilogue != NULL && epilogue->bias != NULL) { + if (block_channels < LW_NHWC_OC_BLOCK) { + float bias_values[LW_NHWC_OC_BLOCK] = {0.0f}; + uint32_t lane; + for (lane = 0u; lane < block_channels; ++lane) { + bias_values[lane] = epilogue->bias[block * LW_NHWC_OC_BLOCK + lane]; + } + bias_lo = _mm256_loadu_ps(bias_values); + bias_hi = _mm256_loadu_ps(bias_values + 8u); + } else { + bias_lo = _mm256_loadu_ps(epilogue->bias + block * LW_NHWC_OC_BLOCK); + bias_hi = _mm256_loadu_ps(epilogue->bias + block * LW_NHWC_OC_BLOCK + 8u); + } + } + for (tap = 0u; tap < LW_CT_TAPS; ++tap) { + __m256 acc_lo[LW_CT_TILE]; + __m256 acc_hi[LW_CT_TILE]; + uint32_t input_channel; + uint32_t pixel; + for (pixel = 0u; pixel < LW_CT_TILE; ++pixel) { + acc_lo[pixel] = bias_lo; + acc_hi[pixel] = bias_hi; + } + for (input_channel = 0u; input_channel < desc->input_channels; ++input_channel) { + const float* weight = packed_weights + + ((((size_t)tap * oc_blocks + block) * desc->input_channels + + input_channel) * LW_NHWC_OC_BLOCK); + __m256 w_lo = _mm256_loadu_ps(weight); + __m256 w_hi = _mm256_loadu_ps(weight + 8u); + for (pixel = 0u; pixel < pixels; ++pixel) { + __m256 value = _mm256_set1_ps( + input_row[(size_t)(input_x + pixel) * desc->input_channels + + input_channel]); + acc_lo[pixel] = _mm256_fmadd_ps(value, w_lo, acc_lo[pixel]); + acc_hi[pixel] = _mm256_fmadd_ps(value, w_hi, acc_hi[pixel]); + } + } + { + uint32_t dy = tap >> 1u; + uint32_t dx = tap & 1u; + float* output_row = dy == 0u ? output_row0 : output_row1; + for (pixel = 0u; pixel < pixels; ++pixel) { + float* destination = output_row + + ((size_t)((input_x + pixel) * 2u + dx) * desc->output_channels) + + block * LW_NHWC_OC_BLOCK; + if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_RELU) { + acc_lo[pixel] = _mm256_max_ps(acc_lo[pixel], _mm256_setzero_ps()); + acc_hi[pixel] = _mm256_max_ps(acc_hi[pixel], _mm256_setzero_ps()); + } + if (block_channels < LW_NHWC_OC_BLOCK) { + float values[LW_NHWC_OC_BLOCK]; + uint32_t lane; + _mm256_storeu_ps(values, acc_lo[pixel]); + _mm256_storeu_ps(values + 8u, acc_hi[pixel]); + for (lane = 0u; lane < block_channels; ++lane) { + destination[lane] = values[lane]; + } + } else { + _mm256_storeu_ps(destination, acc_lo[pixel]); + _mm256_storeu_ps(destination + 8u, acc_hi[pixel]); + } + } + } + } + } +} +#endif + +lw_status lw_avx2_fma_nhwc_convtranspose2x2_s2_f32( + const float* input, const float* packed_weights, const lw_nhwc_epilogue* epilogue, + float* output, const lw_nhwc_convtranspose_desc* desc) { + if (input == NULL || packed_weights == NULL || output == NULL || desc == NULL || + desc->batch == 0u || desc->input_channels == 0u || desc->input_height == 0u || + desc->input_width == 0u || desc->output_channels == 0u || + (desc->output_channels & 7u) != 0u || desc->output_height != desc->input_height * 2u || + desc->output_width != desc->input_width * 2u) { + return LW_STATUS_INVALID_ARGUMENT; + } +#if !LW_CT_X86 + (void)epilogue; + return LW_STATUS_UNSUPPORTED; +#else + uint32_t oc_blocks; + uint32_t batch; + uint32_t input_y; + oc_blocks = (desc->output_channels + LW_NHWC_OC_BLOCK - 1u) / LW_NHWC_OC_BLOCK; + for (batch = 0u; batch < desc->batch; ++batch) { + const float* input_batch = + input + (size_t)batch * desc->input_height * desc->input_width * desc->input_channels; + float* output_batch = output + + (size_t)batch * desc->output_height * desc->output_width * desc->output_channels; + for (input_y = 0u; input_y < desc->input_height; ++input_y) { + const float* input_row = + input_batch + (size_t)input_y * desc->input_width * desc->input_channels; + float* output_row0 = output_batch + + (size_t)(input_y * 2u) * desc->output_width * desc->output_channels; + float* output_row1 = output_row0 + + (size_t)desc->output_width * desc->output_channels; + uint32_t input_x = 0u; + while (input_x < desc->input_width) { + uint32_t pixels = desc->input_width - input_x; + if (pixels > LW_CT_TILE) pixels = LW_CT_TILE; + convtranspose_tile_16(input_row, packed_weights, epilogue, output_row0, + output_row1, desc, oc_blocks, input_x, pixels); + input_x += pixels; + } + } + } + return LW_STATUS_OK; +#endif +} + +#if LW_CT_X86 +LW_CT_TARGET +#endif +lw_status lw_avx2_fma_nhwc_convtranspose2x2_s2_c1_f32( + const float* input, const float* weights, const lw_nhwc_epilogue* epilogue, + float* output, const lw_nhwc_convtranspose_desc* desc) { + if (input == NULL || weights == NULL || output == NULL || desc == NULL || + desc->batch == 0u || desc->input_channels == 0u || desc->input_height == 0u || + desc->input_width == 0u || desc->output_channels != 1u || + desc->output_height != desc->input_height * 2u || + desc->output_width != desc->input_width * 2u) { + return LW_STATUS_INVALID_ARGUMENT; + } +#if !LW_CT_X86 + (void)epilogue; + return LW_STATUS_UNSUPPORTED; +#else + uint32_t batch; + uint32_t input_y; + { + /* Strided per-tap weight columns (ONNX [ic, 1, 2, 2]: w[tap][ic] = + * weights[ic*4+tap]) gathered once into contiguous vectors. The first + * two 8-lane chunks are vectorized; any remainder runs scalar. */ + uint32_t vector_channels = desc->input_channels >= 16u ? 16u : desc->input_channels; + __m256 weight_vectors[LW_CT_TAPS][2]; + uint32_t chunk; + for (chunk = 0u; chunk * 8u < vector_channels; ++chunk) { + uint32_t tap; + for (tap = 0u; tap < LW_CT_TAPS; ++tap) { + float gathered[8]; + uint32_t lane; + for (lane = 0u; lane < 8u; ++lane) { + gathered[lane] = weights[(size_t)(chunk * 8u + lane) * LW_CT_TAPS + tap]; + } + weight_vectors[tap][chunk] = _mm256_loadu_ps(gathered); + } + } + for (batch = 0u; batch < desc->batch; ++batch) { + const float* input_batch = + input + (size_t)batch * desc->input_height * desc->input_width * desc->input_channels; + float* output_batch = output + (size_t)batch * desc->output_height * desc->output_width; + for (input_y = 0u; input_y < desc->input_height; ++input_y) { + const float* input_row = + input_batch + (size_t)input_y * desc->input_width * desc->input_channels; + float* output_row0 = output_batch + (size_t)(input_y * 2u) * desc->output_width; + float* output_row1 = output_row0 + desc->output_width; + uint32_t input_x; + for (input_x = 0u; input_x < desc->input_width; ++input_x) { + const float* pixel = input_row + (size_t)input_x * desc->input_channels; + float sums[LW_CT_TAPS]; + uint32_t tap; + __m256 acc[LW_CT_TAPS]; + for (tap = 0u; tap < LW_CT_TAPS; ++tap) { + acc[tap] = _mm256_setzero_ps(); + } + if (vector_channels >= 8u) { + __m256 in0 = _mm256_loadu_ps(pixel); + for (tap = 0u; tap < LW_CT_TAPS; ++tap) { + acc[tap] = _mm256_fmadd_ps(in0, weight_vectors[tap][0], acc[tap]); + } + } + if (vector_channels >= 16u) { + __m256 in1 = _mm256_loadu_ps(pixel + 8u); + for (tap = 0u; tap < LW_CT_TAPS; ++tap) { + acc[tap] = _mm256_fmadd_ps(in1, weight_vectors[tap][1], acc[tap]); + } + } + /* Channels beyond the vectorized prefix accumulate into + * lane 0 of a scalar sum per tap. */ + if (desc->input_channels > vector_channels) { + float extra[LW_CT_TAPS]; + uint32_t input_channel; + for (tap = 0u; tap < LW_CT_TAPS; ++tap) extra[tap] = 0.0f; + for (input_channel = vector_channels; input_channel < desc->input_channels; + ++input_channel) { + float value = pixel[input_channel]; + for (tap = 0u; tap < LW_CT_TAPS; ++tap) { + extra[tap] += value * weights[(size_t)input_channel * LW_CT_TAPS + tap]; + } + } + for (tap = 0u; tap < LW_CT_TAPS; ++tap) { + float lane_value[8]; + _mm256_storeu_ps(lane_value, acc[tap]); + lane_value[0] += extra[tap]; + acc[tap] = _mm256_loadu_ps(lane_value); + } + } + for (tap = 0u; tap < LW_CT_TAPS; ++tap) { + float lanes[8]; + _mm256_storeu_ps(lanes, acc[tap]); + sums[tap] = lanes[0] + lanes[1] + lanes[2] + lanes[3] + lanes[4] + + lanes[5] + lanes[6] + lanes[7]; + } + if (epilogue != NULL && epilogue->bias != NULL) { + for (tap = 0u; tap < LW_CT_TAPS; ++tap) sums[tap] += epilogue->bias[0]; + } + if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_RELU) { + for (tap = 0u; tap < LW_CT_TAPS; ++tap) { + if (sums[tap] < 0.0f) sums[tap] = 0.0f; + } + } + output_row0[(size_t)input_x * 2u] = sums[0]; + output_row0[(size_t)input_x * 2u + 1u] = sums[1]; + output_row1[(size_t)input_x * 2u] = sums[2]; + output_row1[(size_t)input_x * 2u + 1u] = sums[3]; + } + } + } + return LW_STATUS_OK; + } +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_dense.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_dense.c new file mode 100644 index 00000000..dc3d8f3b --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_dense.c @@ -0,0 +1,893 @@ +#include "nhwc_internal.h" + +#include +#include +#include +#include +#include + +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) +#include +#include "wasm128_epilogue_internal.h" +#define LW_DENSE_WASM128 1 +#else +#define LW_DENSE_WASM128 0 +#endif + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) +#define LW_NHWC_X86 1 +#include +#endif + +static int dense_mul_u64(uint64_t left, uint64_t right, uint64_t* result) { + if (result == NULL || (right != 0u && left > UINT64_MAX / right)) { + return 0; + } + *result = left * right; + return 1; +} + +static int dense_add_u64(uint64_t left, uint64_t right, uint64_t* result) { + if (result == NULL || left > UINT64_MAX - right) { + return 0; + } + *result = left + right; + return 1; +} + +static int dense_align64(uint64_t value, uint64_t* result) { + if (value > UINT64_MAX - 63u) { + return 0; + } + *result = (value + 63u) & ~UINT64_C(63); + return 1; +} + +static int dense_geometry(const lw_nhwc_dense_desc* desc, uint64_t* taps, + uint64_t* k_total, uint64_t* patch_width, + uint64_t* x_tiles) { + uint64_t expected_height; + uint64_t expected_width; + if (desc == NULL || desc->batch == 0u || desc->input_channels == 0u || + desc->input_height == 0u || desc->input_width == 0u || + desc->output_channels == 0u || desc->output_height == 0u || + desc->output_width == 0u || desc->kernel_h == 0u || desc->kernel_w == 0u || + desc->stride_h == 0u || desc->stride_w == 0u) { + return 0; + } + if (!dense_mul_u64(desc->kernel_h, desc->kernel_w, taps) || + !dense_mul_u64(*taps, desc->input_channels, k_total) || + !dense_mul_u64(LW_NHWC_PIXEL_TILE - 1u, desc->stride_w, patch_width) || + !dense_add_u64(*patch_width, desc->kernel_w, patch_width)) { + return 0; + } + if (desc->input_height + (uint64_t)desc->pad_top + desc->pad_bottom < desc->kernel_h || + desc->input_width + (uint64_t)desc->pad_left + desc->pad_right < desc->kernel_w) { + return 0; + } + expected_height = (desc->input_height + (uint64_t)desc->pad_top + desc->pad_bottom - desc->kernel_h) / + desc->stride_h + 1u; + expected_width = (desc->input_width + (uint64_t)desc->pad_left + desc->pad_right - desc->kernel_w) / + desc->stride_w + 1u; + if ((uint64_t)desc->output_height + desc->output_row_offset > expected_height || + expected_width != desc->output_width || + *patch_width > UINT32_MAX || + !dense_add_u64(desc->output_width, LW_NHWC_PIXEL_TILE - 1u, x_tiles)) { + return 0; + } + *x_tiles /= LW_NHWC_PIXEL_TILE; + return *k_total <= UINT32_MAX && *taps <= UINT32_MAX; +} + +int lw_nhwc_dense_scratch_bytes(const lw_nhwc_dense_desc* desc, uint64_t* scratch_bytes) { + uint64_t taps; + uint64_t k_total; + uint64_t patch_width; + uint64_t x_tiles; + uint64_t patch_floats; + uint64_t partial_floats; + uint64_t bytes; + uint64_t aligned; + if (scratch_bytes == NULL || !dense_geometry(desc, &taps, &k_total, &patch_width, &x_tiles)) { + return 0; + } + (void)k_total; + if (!dense_mul_u64(desc->kernel_h, patch_width, &patch_floats) || + !dense_mul_u64(patch_floats, desc->input_channels, &patch_floats) || + !dense_mul_u64(1u, LW_NHWC_PARTIAL_FLOATS, &partial_floats) || + !dense_mul_u64(taps, sizeof(int32_t) * 2u, &bytes) || + !dense_align64(bytes, &aligned) || + !dense_mul_u64(patch_floats, sizeof(float), &patch_floats) || + !dense_add_u64(aligned, patch_floats, &bytes) || + !dense_align64(bytes, &aligned) || + !dense_mul_u64(partial_floats, sizeof(float), &partial_floats) || + !dense_add_u64(aligned, partial_floats, &bytes) || + bytes > SIZE_MAX) { + return 0; + } + *scratch_bytes = bytes; + return 1; +} + +int lw_nhwc_dense_prepared_scratch_bytes(const lw_nhwc_dense_desc* desc, uint64_t* scratch_bytes) { + uint64_t taps; + uint64_t k_total; + uint64_t patch_width; + uint64_t x_tiles; + uint64_t patch_floats; + uint64_t partial_floats; + uint64_t bytes; + uint64_t aligned; + if (scratch_bytes == NULL || !dense_geometry(desc, &taps, &k_total, &patch_width, &x_tiles)) { + return 0; + } + (void)taps; + (void)k_total; + (void)x_tiles; + if (!dense_mul_u64(desc->kernel_h, patch_width, &patch_floats) || + !dense_mul_u64(patch_floats, desc->input_channels, &patch_floats) || + !dense_mul_u64(patch_floats, sizeof(float), &bytes) || + !dense_align64(bytes, &aligned) || + !dense_mul_u64(LW_NHWC_PARTIAL_FLOATS, sizeof(float), &partial_floats) || + !dense_add_u64(aligned, partial_floats, &bytes) || + bytes > SIZE_MAX) { + return 0; + } + *scratch_bytes = bytes; + return 1; +} + +static int dense_prepared_scratch_views(const lw_nhwc_dense_desc* desc, void* scratch, + uint64_t scratch_bytes, float** patch, + float** partial) { + uint64_t required; + uint64_t taps; + uint64_t k_total; + uint64_t patch_width; + uint64_t x_tiles; + uint64_t patch_floats; + uint64_t patch_bytes; + uint64_t partial_bytes; + uint64_t aligned; + if (scratch == NULL || patch == NULL || partial == NULL || + !lw_nhwc_dense_prepared_scratch_bytes(desc, &required) || + scratch_bytes < required || + !dense_geometry(desc, &taps, &k_total, &patch_width, &x_tiles)) { + return 0; + } + (void)taps; + (void)k_total; + (void)x_tiles; + if (!dense_mul_u64(desc->kernel_h, patch_width, &patch_floats) || + !dense_mul_u64(patch_floats, desc->input_channels, &patch_floats) || + !dense_mul_u64(patch_floats, sizeof(float), &patch_bytes) || + !dense_align64(patch_bytes, &aligned) || + !dense_mul_u64(LW_NHWC_PARTIAL_FLOATS, sizeof(float), &partial_bytes) || + !dense_add_u64(aligned, partial_bytes, &partial_bytes)) { + return 0; + } + *patch = (float*)scratch; + *partial = (float*)((uint8_t*)scratch + (size_t)aligned); + return 1; +} + +static int dense_scratch_views(const lw_nhwc_dense_desc* desc, void* scratch, + uint64_t scratch_bytes, int32_t** tap_offsets, + int32_t** patch_offsets, float** patch, float** partial) { + uint64_t required; + uint64_t taps; + uint64_t k_total; + uint64_t patch_width; + uint64_t x_tiles; + uint64_t bytes; + uint64_t aligned; + uint64_t patch_floats; + if (scratch == NULL || !lw_nhwc_dense_scratch_bytes(desc, &required) || + scratch_bytes < required || !dense_geometry(desc, &taps, &k_total, &patch_width, &x_tiles)) { + return 0; + } + (void)k_total; + (void)x_tiles; + if (!dense_mul_u64(taps, sizeof(int32_t) * 2u, &bytes) || + !dense_align64(bytes, &aligned) || + !dense_mul_u64(desc->kernel_h, patch_width, &patch_floats) || + !dense_mul_u64(patch_floats, desc->input_channels, &patch_floats) || + !dense_mul_u64(patch_floats, sizeof(float), &patch_floats)) { + return 0; + } + *tap_offsets = (int32_t*)scratch; + *patch_offsets = (int32_t*)((uint8_t*)scratch + (size_t)taps * sizeof(int32_t)); + *patch = (float*)((uint8_t*)scratch + (size_t)aligned); + if (!dense_align64(aligned + patch_floats, &bytes)) { + return 0; + } + *partial = (float*)((uint8_t*)scratch + (size_t)bytes); + return 1; +} + +static int dense_build_offsets(const lw_nhwc_dense_desc* desc, int32_t* tap_offsets, + int32_t* patch_offsets, uint32_t patch_width) { + uint32_t tap = 0u; + uint32_t ky; + for (ky = 0u; ky < desc->kernel_h; ++ky) { + uint32_t kx; + for (kx = 0u; kx < desc->kernel_w; ++kx) { + uint64_t input_offset = ((uint64_t)ky * desc->input_width + kx) * + desc->input_channels; + uint64_t patch_offset = ((uint64_t)ky * patch_width + kx) * + desc->input_channels; + if (input_offset > INT32_MAX || patch_offset > INT32_MAX) { + return 0; + } + tap_offsets[tap] = (int32_t)input_offset; + patch_offsets[tap] = (int32_t)patch_offset; + ++tap; + } + } + return 1; +} + +static void dense_gather_patch(const float* input, float* patch, + const lw_nhwc_dense_desc* desc, int32_t iy0, + int32_t ix0, uint32_t patch_width) { + uint64_t patch_elements = (uint64_t)desc->kernel_h * patch_width * desc->input_channels; + uint32_t py; + memset(patch, 0, (size_t)patch_elements * sizeof(float)); + for (py = 0u; py < desc->kernel_h; ++py) { + int32_t iy = iy0 + (int32_t)py; + uint32_t px; + if (iy < 0 || iy >= (int32_t)desc->input_height) { + continue; + } + for (px = 0u; px < patch_width; ++px) { + int32_t ix = ix0 + (int32_t)px; + const float* source; + float* destination; + if (ix < 0 || ix >= (int32_t)desc->input_width) { + continue; + } + source = input + (((size_t)iy * desc->input_width + (uint32_t)ix) * + desc->input_channels); + destination = patch + (((size_t)py * patch_width + px) * desc->input_channels); + memcpy(destination, source, (size_t)desc->input_channels * sizeof(float)); + } + } +} + +static float dense_apply_activation(float value, uint16_t activation) { + if (activation == LW_NHWC_ACT_RELU) { + return value > 0.0f ? value : 0.0f; + } + if (activation == LW_NHWC_ACT_HARDSWISH) { + float gate = value + 3.0f; + if (gate < 0.0f) gate = 0.0f; + if (gate > 6.0f) gate = 6.0f; + return value * gate * (1.0f / 6.0f); + } + if (activation == LW_NHWC_ACT_GELU) { + return lw_nhwc_gelu_scalar_exact_f32(value); + } + return value; +} + +#if !defined(LW_NHWC_X86) && !LW_DENSE_WASM128 +static void dense_tile_scalar(const float* const* row_ptrs, uint32_t rows, + const int32_t* offsets, const int32_t* direct_offsets, uint32_t taps, + const float* packed, uint32_t k0, uint32_t k_end, + uint32_t k_total, const lw_nhwc_epilogue* epilogue, + float* output, uint32_t output_stride, + uint32_t block_channels, float* partial) { + float sums[LW_NHWC_PIXEL_TILE][LW_NHWC_OC_BLOCK]; + uint32_t row; + uint32_t lane; + if (k0 == 0u) { + for (row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + for (lane = 0u; lane < LW_NHWC_OC_BLOCK; ++lane) { + sums[row][lane] = epilogue != NULL && epilogue->bias != NULL + ? epilogue->bias[lane] : 0.0f; + } + } + } else { + memcpy(sums, partial, sizeof(sums)); + } + for (uint32_t k = k0; k < k_end; ++k) { + int32_t input_offset; + if (direct_offsets != NULL) input_offset = direct_offsets[k]; + else { + uint32_t ic = k / taps; + uint32_t tap = k - ic * taps; + input_offset = offsets[tap] + (int32_t)ic; + } + const float* weights = packed + (size_t)(k - k0) * LW_NHWC_OC_BLOCK; + for (row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + float value = row_ptrs[row][input_offset]; + for (lane = 0u; lane < LW_NHWC_OC_BLOCK; ++lane) { + sums[row][lane] += value * weights[lane]; + } + } + } + if (k_end < k_total) { + memcpy(partial, sums, sizeof(sums)); + return; + } + for (row = 0u; row < rows; ++row) { + float* destination = output + (size_t)row * output_stride; + const float* residual = epilogue == NULL || epilogue->residual == NULL + ? NULL : epilogue->residual + (size_t)row * output_stride; + for (lane = 0u; lane < block_channels; ++lane) { + float value = sums[row][lane]; + if (epilogue != NULL && epilogue->post_bias != NULL) { + value += epilogue->post_bias[lane]; + } + if (residual != NULL) { + value += residual[lane]; + } + destination[lane] = dense_apply_activation(value, + epilogue == NULL ? LW_NHWC_ACT_NONE : epilogue->activation); + } + } +} + +#endif + +#if LW_DENSE_WASM128 +static void dense_tile_wasm128(const float* const* row_ptrs, uint32_t rows, + const int32_t* offsets, const int32_t* direct_offsets, + uint32_t taps, const float* packed, uint32_t k0, + uint32_t k_end, uint32_t k_total, + const lw_nhwc_epilogue* epilogue, float* output, + uint32_t output_stride, uint32_t block_channels, + float* partial) { + v128_t sums[LW_NHWC_PIXEL_TILE][LW_NHWC_OC_BLOCK / 4u]; + float bias_values[LW_NHWC_OC_BLOCK] = {0.0f}; + if (k0 == 0u && epilogue != NULL && epilogue->bias != NULL) { + memcpy(bias_values, epilogue->bias, (size_t)block_channels * sizeof(float)); + } + for (uint32_t row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + for (uint32_t lane = 0u; lane < LW_NHWC_OC_BLOCK / 4u; ++lane) { + sums[row][lane] = wasm_v128_load((k0 == 0u ? bias_values : + partial + (size_t)row * LW_NHWC_OC_BLOCK) + lane * 4u); + } + } + for (uint32_t k = k0; k < k_end; ++k) { + int32_t input_offset; + if (direct_offsets != NULL) input_offset = direct_offsets[k]; + else { + uint32_t ic = k / taps; + input_offset = offsets[k - ic * taps] + (int32_t)ic; + } + const float* weights = packed + (size_t)(k - k0) * LW_NHWC_OC_BLOCK; + v128_t weight0 = wasm_v128_load(weights); + v128_t weight1 = wasm_v128_load(weights + 4u); + v128_t weight2 = wasm_v128_load(weights + 8u); + v128_t weight3 = wasm_v128_load(weights + 12u); + for (uint32_t row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + v128_t value = wasm_f32x4_splat(row_ptrs[row][input_offset]); + sums[row][0] = wasm_f32x4_add(sums[row][0], wasm_f32x4_mul(value, weight0)); + sums[row][1] = wasm_f32x4_add(sums[row][1], wasm_f32x4_mul(value, weight1)); + sums[row][2] = wasm_f32x4_add(sums[row][2], wasm_f32x4_mul(value, weight2)); + sums[row][3] = wasm_f32x4_add(sums[row][3], wasm_f32x4_mul(value, weight3)); + } + } + if (k_end < k_total) { + for (uint32_t row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + for (uint32_t lane = 0u; lane < LW_NHWC_OC_BLOCK / 4u; ++lane) { + wasm_v128_store(partial + (size_t)row * LW_NHWC_OC_BLOCK + lane * 4u, + sums[row][lane]); + } + } + return; + } + for (uint32_t row = 0u; row < rows; ++row) { + float* destination = output + (size_t)row * output_stride; + const float* residual = epilogue == NULL || epilogue->residual == NULL + ? NULL : epilogue->residual + (size_t)row * output_stride; + if (block_channels == LW_NHWC_OC_BLOCK) { + v128_t values[4] = { + sums[row][0], sums[row][1], sums[row][2], sums[row][3] + }; + lw_wasm128_apply_epilogue_oc16(values, epilogue, 0u, residual, 0); + for (uint32_t group = 0u; group < 4u; ++group) { + wasm_v128_store(destination + group * 4u, values[group]); + } + } else { + float values[LW_NHWC_OC_BLOCK]; + for (uint32_t group = 0u; group < 4u; ++group) { + wasm_v128_store(values + group * 4u, sums[row][group]); + } + for (uint32_t lane = 0u; lane < block_channels; ++lane) { + float value = values[lane]; + if (epilogue != NULL && epilogue->post_bias != NULL) + value += epilogue->post_bias[lane]; + if (residual != NULL) value += residual[lane]; + destination[lane] = dense_apply_activation(value, + epilogue == NULL ? LW_NHWC_ACT_NONE : epilogue->activation); + } + } + } +} +#endif + +#if defined(LW_NHWC_X86) +#if defined(__GNUC__) || defined(__clang__) +#define LW_NHWC_DENSE_AVX2 __attribute__((target("avx2,fma"))) +#else +#define LW_NHWC_DENSE_AVX2 +#endif + +LW_NHWC_DENSE_AVX2 +static void dense_tile_avx2(const float* const* row_ptrs, uint32_t rows, + const int32_t* offsets, const int32_t* direct_offsets, uint32_t taps, + const float* packed, uint32_t k0, uint32_t k_end, + uint32_t k_total, const lw_nhwc_epilogue* epilogue, + float* output, uint32_t output_stride, + uint32_t block_channels, float* partial) { + __m256 lo[LW_NHWC_PIXEL_TILE]; + __m256 hi[LW_NHWC_PIXEL_TILE]; + __m256 zero = _mm256_setzero_ps(); + uint32_t row; + if (k0 == 0u) { + __m256 bias_lo = zero; + __m256 bias_hi = zero; + if (epilogue != NULL && epilogue->bias != NULL) { + if (block_channels < LW_NHWC_OC_BLOCK) { + float bias_values[LW_NHWC_OC_BLOCK] = {0.0f}; + for (uint32_t lane = 0u; lane < block_channels; ++lane) { + bias_values[lane] = epilogue->bias[lane]; + } + bias_lo = _mm256_loadu_ps(bias_values); + bias_hi = _mm256_loadu_ps(bias_values + 8u); + } else { + bias_lo = _mm256_loadu_ps(epilogue->bias); + bias_hi = _mm256_loadu_ps(epilogue->bias + 8u); + } + } + for (row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + lo[row] = bias_lo; + hi[row] = bias_hi; + } + } else { + for (row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + lo[row] = _mm256_loadu_ps(partial + row * 16u); + hi[row] = _mm256_loadu_ps(partial + row * 16u + 8u); + } + } + for (uint32_t k = k0; k < k_end; ++k) { + int32_t input_offset; + if (direct_offsets != NULL) input_offset = direct_offsets[k]; + else { + uint32_t ic = k / taps; + uint32_t tap = k - ic * taps; + input_offset = offsets[tap] + (int32_t)ic; + } + const float* weights = packed + (size_t)(k - k0) * 16u; + __m256 w0 = _mm256_loadu_ps(weights); + __m256 w1 = _mm256_loadu_ps(weights + 8u); + for (row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + __m256 value = _mm256_set1_ps(row_ptrs[row][input_offset]); + lo[row] = _mm256_fmadd_ps(value, w0, lo[row]); + hi[row] = _mm256_fmadd_ps(value, w1, hi[row]); + } + } + if (k_end < k_total) { + for (row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + _mm256_storeu_ps(partial + row * 16u, lo[row]); + _mm256_storeu_ps(partial + row * 16u + 8u, hi[row]); + } + return; + } + for (row = 0u; row < rows; ++row) { + float* destination = output + (size_t)row * output_stride; + if (block_channels < LW_NHWC_OC_BLOCK) { + float values[LW_NHWC_OC_BLOCK]; + _mm256_storeu_ps(values, lo[row]); + _mm256_storeu_ps(values + 8u, hi[row]); + for (uint32_t lane = 0u; lane < block_channels; ++lane) { + float value = values[lane]; + if (epilogue != NULL && epilogue->post_bias != NULL) { + value += epilogue->post_bias[lane]; + } + if (epilogue != NULL && epilogue->residual != NULL) { + value += epilogue->residual[(size_t)row * output_stride + lane]; + } + values[lane] = value; + } + if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_GELU) { + /* Match the standalone pass layout: full 8-lane groups use + * the vector polynomial, the sub-8 tail the scalar erff + * helper. block_channels < 16, so at most one vector group. */ + uint32_t vector_end = block_channels & ~7u; + uint32_t lane; + if (vector_end >= 8u) { + _mm256_storeu_ps(values, + lw_nhwc_avx2_gelu_vector_exact_f32(_mm256_loadu_ps(values))); + } + for (lane = vector_end; lane < block_channels; ++lane) { + values[lane] = lw_nhwc_gelu_scalar_exact_f32(values[lane]); + } + for (lane = 0u; lane < block_channels; ++lane) { + destination[lane] = values[lane]; + } + continue; + } + for (uint32_t lane = 0u; lane < block_channels; ++lane) { + destination[lane] = dense_apply_activation(values[lane], + epilogue == NULL ? LW_NHWC_ACT_NONE : epilogue->activation); + } + continue; + } + if (epilogue != NULL && epilogue->post_bias != NULL) { + lo[row] = _mm256_add_ps(lo[row], _mm256_loadu_ps(epilogue->post_bias)); + hi[row] = _mm256_add_ps(hi[row], _mm256_loadu_ps(epilogue->post_bias + 8u)); + } + if (epilogue != NULL && epilogue->residual != NULL) { + __m256 residual_lo = _mm256_loadu_ps(epilogue->residual + row * output_stride); + __m256 residual_hi = _mm256_loadu_ps(epilogue->residual + row * output_stride + 8u); + lo[row] = _mm256_add_ps(lo[row], residual_lo); + hi[row] = _mm256_add_ps(hi[row], residual_hi); + } + if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_RELU) { + lo[row] = _mm256_max_ps(lo[row], zero); + hi[row] = _mm256_max_ps(hi[row], zero); + } else if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_HARDSWISH) { + /* Match dense_apply_activation: gate = clamp(x + 3, 0, 6); + * y = x * gate * (1/6). */ + const __m256 three = _mm256_set1_ps(3.0f); + const __m256 six = _mm256_set1_ps(6.0f); + const __m256 inverse_six = _mm256_set1_ps(1.0f / 6.0f); + __m256 gate_lo = _mm256_min_ps(_mm256_max_ps(_mm256_add_ps(lo[row], three), zero), six); + __m256 gate_hi = _mm256_min_ps(_mm256_max_ps(_mm256_add_ps(hi[row], three), zero), six); + lo[row] = _mm256_mul_ps(_mm256_mul_ps(lo[row], gate_lo), inverse_six); + hi[row] = _mm256_mul_ps(_mm256_mul_ps(hi[row], gate_hi), inverse_six); + } else if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_GELU) { + lo[row] = lw_nhwc_avx2_gelu_vector_exact_f32(lo[row]); + hi[row] = lw_nhwc_avx2_gelu_vector_exact_f32(hi[row]); + } + _mm256_storeu_ps(destination, lo[row]); + _mm256_storeu_ps(destination + 8u, hi[row]); + } +} + +/* Fully unrolled variant of dense_tile_avx2 for full tiles (6 rows, 16 + * channels): named YMM accumulators instead of arrays so the compiler keeps + * them in registers (the array form spills). Same bias/partial + * initialisation, same ascending k FMA order, same epilogue, so results are + * bit-identical. Partial tiles or partial channel blocks delegate to + * dense_tile_avx2. */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((noinline)) +#elif defined(_MSC_VER) +__declspec(noinline) +#endif +LW_NHWC_DENSE_AVX2 +static void dense_tile_avx2_unrolled(const float* const* row_ptrs, uint32_t rows, + const int32_t* offsets, const int32_t* direct_offsets, uint32_t taps, + const float* packed, uint32_t k0, uint32_t k_end, + uint32_t k_total, const lw_nhwc_epilogue* epilogue, + float* output, uint32_t output_stride, + uint32_t block_channels, float* partial) { + __m256 lo0, lo1, lo2, lo3, lo4, lo5; + __m256 hi0, hi1, hi2, hi3, hi4, hi5; + __m256 zero = _mm256_setzero_ps(); + uint32_t k; + if (rows < LW_NHWC_PIXEL_TILE || block_channels < LW_NHWC_OC_BLOCK) { + dense_tile_avx2(row_ptrs, rows, offsets, direct_offsets, taps, packed, + k0, k_end, k_total, epilogue, output, output_stride, + block_channels, partial); + return; + } + if (k0 == 0u) { + __m256 bias_lo = zero; + __m256 bias_hi = zero; + if (epilogue != NULL && epilogue->bias != NULL) { + bias_lo = _mm256_loadu_ps(epilogue->bias); + bias_hi = _mm256_loadu_ps(epilogue->bias + 8u); + } + lo0 = bias_lo; lo1 = bias_lo; lo2 = bias_lo; + lo3 = bias_lo; lo4 = bias_lo; lo5 = bias_lo; + hi0 = bias_hi; hi1 = bias_hi; hi2 = bias_hi; + hi3 = bias_hi; hi4 = bias_hi; hi5 = bias_hi; + } else { + lo0 = _mm256_loadu_ps(partial); + hi0 = _mm256_loadu_ps(partial + 8u); + lo1 = _mm256_loadu_ps(partial + 16u); + hi1 = _mm256_loadu_ps(partial + 24u); + lo2 = _mm256_loadu_ps(partial + 32u); + hi2 = _mm256_loadu_ps(partial + 40u); + lo3 = _mm256_loadu_ps(partial + 48u); + hi3 = _mm256_loadu_ps(partial + 56u); + lo4 = _mm256_loadu_ps(partial + 64u); + hi4 = _mm256_loadu_ps(partial + 72u); + lo5 = _mm256_loadu_ps(partial + 80u); + hi5 = _mm256_loadu_ps(partial + 88u); + } + for (k = k0; k < k_end; ++k) { + int32_t input_offset; + const float* weights; + __m256 w0; + __m256 w1; + __m256 value; + if (direct_offsets != NULL) input_offset = direct_offsets[k]; + else { + uint32_t ic = k / taps; + uint32_t tap = k - ic * taps; + input_offset = offsets[tap] + (int32_t)ic; + } + weights = packed + (size_t)(k - k0) * 16u; + w0 = _mm256_loadu_ps(weights); + w1 = _mm256_loadu_ps(weights + 8u); + value = _mm256_set1_ps(row_ptrs[0][input_offset]); + lo0 = _mm256_fmadd_ps(value, w0, lo0); + hi0 = _mm256_fmadd_ps(value, w1, hi0); + value = _mm256_set1_ps(row_ptrs[1][input_offset]); + lo1 = _mm256_fmadd_ps(value, w0, lo1); + hi1 = _mm256_fmadd_ps(value, w1, hi1); + value = _mm256_set1_ps(row_ptrs[2][input_offset]); + lo2 = _mm256_fmadd_ps(value, w0, lo2); + hi2 = _mm256_fmadd_ps(value, w1, hi2); + value = _mm256_set1_ps(row_ptrs[3][input_offset]); + lo3 = _mm256_fmadd_ps(value, w0, lo3); + hi3 = _mm256_fmadd_ps(value, w1, hi3); + value = _mm256_set1_ps(row_ptrs[4][input_offset]); + lo4 = _mm256_fmadd_ps(value, w0, lo4); + hi4 = _mm256_fmadd_ps(value, w1, hi4); + value = _mm256_set1_ps(row_ptrs[5][input_offset]); + lo5 = _mm256_fmadd_ps(value, w0, lo5); + hi5 = _mm256_fmadd_ps(value, w1, hi5); + } + if (k_end < k_total) { + _mm256_storeu_ps(partial, lo0); + _mm256_storeu_ps(partial + 8u, hi0); + _mm256_storeu_ps(partial + 16u, lo1); + _mm256_storeu_ps(partial + 24u, hi1); + _mm256_storeu_ps(partial + 32u, lo2); + _mm256_storeu_ps(partial + 40u, hi2); + _mm256_storeu_ps(partial + 48u, lo3); + _mm256_storeu_ps(partial + 56u, hi3); + _mm256_storeu_ps(partial + 64u, lo4); + _mm256_storeu_ps(partial + 72u, hi4); + _mm256_storeu_ps(partial + 80u, lo5); + _mm256_storeu_ps(partial + 88u, hi5); + return; + } + /* Full tile, full 16-channel block: reuse the exact vector epilogue of + * dense_tile_avx2 (runs once per tile, so array indexing is harmless). */ + { + __m256 lo[LW_NHWC_PIXEL_TILE]; + __m256 hi[LW_NHWC_PIXEL_TILE]; + uint32_t row; + lo[0] = lo0; lo[1] = lo1; lo[2] = lo2; + lo[3] = lo3; lo[4] = lo4; lo[5] = lo5; + hi[0] = hi0; hi[1] = hi1; hi[2] = hi2; + hi[3] = hi3; hi[4] = hi4; hi[5] = hi5; + for (row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + float* destination = output + (size_t)row * output_stride; + if (epilogue != NULL && epilogue->post_bias != NULL) { + lo[row] = _mm256_add_ps(lo[row], _mm256_loadu_ps(epilogue->post_bias)); + hi[row] = _mm256_add_ps(hi[row], _mm256_loadu_ps(epilogue->post_bias + 8u)); + } + if (epilogue != NULL && epilogue->residual != NULL) { + __m256 residual_lo = _mm256_loadu_ps(epilogue->residual + row * output_stride); + __m256 residual_hi = _mm256_loadu_ps(epilogue->residual + row * output_stride + 8u); + lo[row] = _mm256_add_ps(lo[row], residual_lo); + hi[row] = _mm256_add_ps(hi[row], residual_hi); + } + if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_RELU) { + lo[row] = _mm256_max_ps(lo[row], zero); + hi[row] = _mm256_max_ps(hi[row], zero); + } else if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_HARDSWISH) { + const __m256 three = _mm256_set1_ps(3.0f); + const __m256 six = _mm256_set1_ps(6.0f); + const __m256 inverse_six = _mm256_set1_ps(1.0f / 6.0f); + __m256 gate_lo = _mm256_min_ps(_mm256_max_ps(_mm256_add_ps(lo[row], three), zero), six); + __m256 gate_hi = _mm256_min_ps(_mm256_max_ps(_mm256_add_ps(hi[row], three), zero), six); + lo[row] = _mm256_mul_ps(_mm256_mul_ps(lo[row], gate_lo), inverse_six); + hi[row] = _mm256_mul_ps(_mm256_mul_ps(hi[row], gate_hi), inverse_six); + } else if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_GELU) { + lo[row] = lw_nhwc_avx2_gelu_vector_exact_f32(lo[row]); + hi[row] = lw_nhwc_avx2_gelu_vector_exact_f32(hi[row]); + } + _mm256_storeu_ps(destination, lo[row]); + _mm256_storeu_ps(destination + 8u, hi[row]); + } + } +} +#endif + + +lw_status lw_avx2_fma_nhwc_dense_f32(const float* input, const float* packed_weights, + const lw_nhwc_epilogue* epilogue, float* output, + const lw_nhwc_dense_desc* desc, void* scratch, + uint64_t scratch_bytes) { + int32_t* tap_offsets; + int32_t* patch_offsets; + float* patch; + float* partial; + uint64_t taps64; + uint64_t k_total64; + uint64_t patch_width64; + uint64_t x_tiles64; + uint32_t kc; + if (input == NULL || packed_weights == NULL || output == NULL || desc == NULL || + !dense_geometry(desc, &taps64, &k_total64, &patch_width64, &x_tiles64) || + !dense_scratch_views(desc, scratch, scratch_bytes, &tap_offsets, &patch_offsets, + &patch, &partial) || + !dense_build_offsets(desc, tap_offsets, patch_offsets, (uint32_t)patch_width64)) { + return LW_STATUS_INVALID_ARGUMENT; + } + kc = desc->dense_kc == 0u ? (uint32_t)k_total64 : desc->dense_kc; + if (kc == 0u) return LW_STATUS_INVALID_ARGUMENT; + for (uint32_t batch = 0u; batch < desc->batch; ++batch) { + const float* input_batch = input + (size_t)batch * desc->input_height * + desc->input_width * desc->input_channels; + float* output_batch = output + (size_t)batch * desc->output_height * + desc->output_width * desc->output_channels; + for (uint32_t oy = 0u; oy < desc->output_height; ++oy) { + int32_t iy0 = (int32_t)((uint64_t)(oy + desc->output_row_offset) * + desc->stride_h) - (int32_t)desc->pad_top; + for (uint32_t tile = 0u; tile < (uint32_t)x_tiles64; ++tile) { + uint32_t ox = tile * LW_NHWC_PIXEL_TILE; + uint32_t rows = desc->output_width - ox; + int32_t ix0 = (int32_t)((uint64_t)ox * desc->stride_w) - (int32_t)desc->pad_left; + int interior = iy0 >= 0 && iy0 + (int32_t)desc->kernel_h <= (int32_t)desc->input_height && + ix0 >= 0 && ix0 + (int32_t)((LW_NHWC_PIXEL_TILE - 1u) * desc->stride_w + desc->kernel_w) <= + (int32_t)desc->input_width; + const float* row_ptrs[LW_NHWC_PIXEL_TILE]; + if (rows > LW_NHWC_PIXEL_TILE) rows = LW_NHWC_PIXEL_TILE; + if (interior) { + for (uint32_t row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + row_ptrs[row] = input_batch + (((size_t)iy0 * desc->input_width + + (uint32_t)(ix0 + (int32_t)row * desc->stride_w)) * desc->input_channels); + } + } else { + dense_gather_patch(input_batch, patch, desc, iy0, ix0, (uint32_t)patch_width64); + for (uint32_t row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + row_ptrs[row] = patch + (size_t)row * desc->stride_w * desc->input_channels; + } + } + for (uint32_t block = 0u; block < (desc->output_channels + LW_NHWC_OC_BLOCK - 1u) / LW_NHWC_OC_BLOCK; ++block) { + uint32_t block_channels = desc->output_channels - block * LW_NHWC_OC_BLOCK; + if (block_channels > LW_NHWC_OC_BLOCK) block_channels = LW_NHWC_OC_BLOCK; + const float* packed_block = packed_weights + + (size_t)block * k_total64 * LW_NHWC_OC_BLOCK; + float* destination = output_batch + ((size_t)oy * desc->output_width + ox) * + desc->output_channels + block * LW_NHWC_OC_BLOCK; + lw_nhwc_epilogue local_epilogue = epilogue == NULL + ? (lw_nhwc_epilogue){0} : *epilogue; + if (local_epilogue.bias != NULL) local_epilogue.bias += block * LW_NHWC_OC_BLOCK; + if (local_epilogue.post_bias != NULL) local_epilogue.post_bias += block * LW_NHWC_OC_BLOCK; + if (local_epilogue.residual != NULL) { + local_epilogue.residual += ((size_t)batch * desc->output_height * desc->output_width + + (size_t)oy * desc->output_width + ox) * desc->output_channels + + block * LW_NHWC_OC_BLOCK; + } + for (uint32_t k0 = 0u; k0 < (uint32_t)k_total64; k0 += kc) { + uint32_t k_end = k0 + kc; + if (k_end > (uint32_t)k_total64) k_end = (uint32_t)k_total64; +#if defined(LW_NHWC_X86) + dense_tile_avx2_unrolled(row_ptrs, rows, + interior ? tap_offsets : patch_offsets, NULL, (uint32_t)taps64, + packed_block + (size_t)k0 * LW_NHWC_OC_BLOCK, k0, k_end, + (uint32_t)k_total64, &local_epilogue, destination, + desc->output_channels, block_channels, partial); +#elif LW_DENSE_WASM128 + dense_tile_wasm128(row_ptrs, rows, + interior ? tap_offsets : patch_offsets, NULL, (uint32_t)taps64, + packed_block + (size_t)k0 * LW_NHWC_OC_BLOCK, k0, k_end, + (uint32_t)k_total64, &local_epilogue, destination, + desc->output_channels, block_channels, partial); +#else + dense_tile_scalar(row_ptrs, rows, + interior ? tap_offsets : patch_offsets, NULL, (uint32_t)taps64, + packed_block + (size_t)k0 * LW_NHWC_OC_BLOCK, k0, k_end, + (uint32_t)k_total64, &local_epilogue, destination, + desc->output_channels, block_channels, partial); +#endif + } + } + } + } + } + return LW_STATUS_OK; +} + +lw_status lw_avx2_fma_nhwc_dense_prepared_f32(const float* input, + const float* packed_weights, + const int32_t* input_offsets_k, + const int32_t* patch_offsets_k, + const lw_nhwc_epilogue* epilogue, + float* output, + const lw_nhwc_dense_desc* desc, + void* scratch, + uint64_t scratch_bytes) { + float* patch; + float* partial; + uint64_t taps64; + uint64_t k_total64; + uint64_t patch_width64; + uint64_t x_tiles64; + uint32_t kc; + if (input == NULL || packed_weights == NULL || input_offsets_k == NULL || + patch_offsets_k == NULL || output == NULL || desc == NULL || + !dense_geometry(desc, &taps64, &k_total64, &patch_width64, &x_tiles64) || + !dense_prepared_scratch_views(desc, scratch, scratch_bytes, &patch, &partial)) { + return LW_STATUS_INVALID_ARGUMENT; + } + kc = desc->dense_kc == 0u ? (uint32_t)k_total64 : desc->dense_kc; + if (kc == 0u) return LW_STATUS_INVALID_ARGUMENT; + for (uint32_t batch = 0u; batch < desc->batch; ++batch) { + const float* input_batch = input + (size_t)batch * desc->input_height * + desc->input_width * desc->input_channels; + float* output_batch = output + (size_t)batch * desc->output_height * + desc->output_width * desc->output_channels; + for (uint32_t oy = 0u; oy < desc->output_height; ++oy) { + int32_t iy0 = (int32_t)((uint64_t)(oy + desc->output_row_offset) * + desc->stride_h) - (int32_t)desc->pad_top; + for (uint32_t tile = 0u; tile < (uint32_t)x_tiles64; ++tile) { + uint32_t ox = tile * LW_NHWC_PIXEL_TILE; + uint32_t rows = desc->output_width - ox; + int32_t ix0 = (int32_t)((uint64_t)ox * desc->stride_w) - (int32_t)desc->pad_left; + int interior = iy0 >= 0 && + iy0 + (int32_t)desc->kernel_h <= (int32_t)desc->input_height && + ix0 >= 0 && + ix0 + (int32_t)((LW_NHWC_PIXEL_TILE - 1u) * desc->stride_w + desc->kernel_w) <= + (int32_t)desc->input_width; + const float* row_ptrs[LW_NHWC_PIXEL_TILE]; + const int32_t* direct_offsets = interior ? input_offsets_k : patch_offsets_k; + if (rows > LW_NHWC_PIXEL_TILE) rows = LW_NHWC_PIXEL_TILE; + if (interior) { + for (uint32_t row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + row_ptrs[row] = input_batch + (((size_t)iy0 * desc->input_width + + (uint32_t)(ix0 + (int32_t)row * desc->stride_w)) * desc->input_channels); + } + } else { + dense_gather_patch(input_batch, patch, desc, iy0, ix0, (uint32_t)patch_width64); + for (uint32_t row = 0u; row < LW_NHWC_PIXEL_TILE; ++row) { + row_ptrs[row] = patch + (size_t)row * desc->stride_w * desc->input_channels; + } + } + for (uint32_t block = 0u; block < (desc->output_channels + LW_NHWC_OC_BLOCK - 1u) / LW_NHWC_OC_BLOCK; ++block) { + uint32_t block_channels = desc->output_channels - block * LW_NHWC_OC_BLOCK; + if (block_channels > LW_NHWC_OC_BLOCK) block_channels = LW_NHWC_OC_BLOCK; + const float* packed_block = packed_weights + + (size_t)block * k_total64 * LW_NHWC_OC_BLOCK; + float* destination = output_batch + ((size_t)oy * desc->output_width + ox) * + desc->output_channels + block * LW_NHWC_OC_BLOCK; + lw_nhwc_epilogue local_epilogue = epilogue == NULL + ? (lw_nhwc_epilogue){0} : *epilogue; + if (local_epilogue.bias != NULL) local_epilogue.bias += block * LW_NHWC_OC_BLOCK; + if (local_epilogue.post_bias != NULL) local_epilogue.post_bias += block * LW_NHWC_OC_BLOCK; + if (local_epilogue.residual != NULL) { + local_epilogue.residual += ((size_t)batch * desc->output_height * desc->output_width + + (size_t)oy * desc->output_width + ox) * desc->output_channels + + block * LW_NHWC_OC_BLOCK; + } + for (uint32_t k0 = 0u; k0 < (uint32_t)k_total64; k0 += kc) { + uint32_t k_end = k0 + kc; + if (k_end > (uint32_t)k_total64) k_end = (uint32_t)k_total64; +#if defined(LW_NHWC_X86) + dense_tile_avx2_unrolled(row_ptrs, rows, NULL, direct_offsets, + (uint32_t)taps64, + packed_block + (size_t)k0 * LW_NHWC_OC_BLOCK, k0, k_end, + (uint32_t)k_total64, &local_epilogue, destination, + desc->output_channels, block_channels, partial); +#elif LW_DENSE_WASM128 + dense_tile_wasm128(row_ptrs, rows, NULL, direct_offsets, + (uint32_t)taps64, + packed_block + (size_t)k0 * LW_NHWC_OC_BLOCK, k0, k_end, + (uint32_t)k_total64, &local_epilogue, destination, + desc->output_channels, block_channels, partial); +#else + dense_tile_scalar(row_ptrs, rows, NULL, direct_offsets, + (uint32_t)taps64, + packed_block + (size_t)k0 * LW_NHWC_OC_BLOCK, k0, k_end, + (uint32_t)k_total64, &local_epilogue, destination, + desc->output_channels, block_channels, partial); +#endif + } + } + } + } + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_depthwise.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_depthwise.c new file mode 100644 index 00000000..b9d5f2eb --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_depthwise.c @@ -0,0 +1,305 @@ +#include "nhwc_internal.h" + +#include +#include + +#if defined(__EMSCRIPTEN__) && defined(LW_WASM_REC_SIMD_KERNELS) +#include +#define LW_DW_WASM128 1 +#else +#define LW_DW_WASM128 0 +#endif + +#if !defined(LW_NHWC_FORCE_PORTABLE) && \ + (defined(_M_X64) || defined(__x86_64__) || defined(_M_IX86) || defined(__i386__)) +#include +#define LW_DW_X86 1 +#else +#define LW_DW_X86 0 +#endif + +#if LW_DW_X86 && (defined(__GNUC__) || defined(__clang__)) +#define LW_DW_TARGET __attribute__((target("avx2,fma"))) +#else +#define LW_DW_TARGET +#endif + +#if LW_DW_X86 +LW_DW_TARGET +static void depthwise_block32(const float* input, const float* weights, const float* bias, + float* output, const lw_nhwc_depthwise_desc* desc, + uint32_t output_y, uint32_t output_x, uint32_t channel_base, uint32_t vector_count) { + __m256 sums[4]; + int32_t input_y0 = (int32_t)((uint64_t)(output_y + desc->output_row_offset) * desc->stride_h) - + (int32_t)desc->pad_top; + int32_t input_x0 = (int32_t)((uint64_t)output_x * desc->stride_w) - (int32_t)desc->pad_left; + for (uint32_t lane = 0u; lane < vector_count; ++lane) { + sums[lane] = bias == NULL ? _mm256_setzero_ps() : + _mm256_loadu_ps(bias + channel_base + lane * 8u); + } + uint32_t tap = 0u; + for (uint32_t ky = 0u; ky < desc->kernel_h; ++ky) { + int32_t iy = input_y0 + (int32_t)ky; + for (uint32_t kx = 0u; kx < desc->kernel_w; ++kx, ++tap) { + int32_t ix = input_x0 + (int32_t)kx; + if (iy < 0 || iy >= (int32_t)desc->input_height || ix < 0 || + ix >= (int32_t)desc->input_width) continue; + const float* source = input + (((size_t)iy * desc->input_width + (uint32_t)ix) * + desc->channels + channel_base); + const float* weight = weights + (size_t)tap * LW_NHWC_DEPTHWISE_BLOCK; + for (uint32_t lane = 0u; lane < vector_count; ++lane) { + sums[lane] = _mm256_fmadd_ps(_mm256_loadu_ps(source + lane * 8u), + _mm256_loadu_ps(weight + lane * 8u), sums[lane]); + } + } + } + float* destination = output + (((size_t)output_y * desc->output_width + output_x) * + desc->channels + channel_base); + if (desc->post_bias != NULL) { + for (uint32_t lane = 0u; lane < vector_count; ++lane) { + sums[lane] = _mm256_add_ps(sums[lane], + _mm256_loadu_ps(desc->post_bias + channel_base + lane * 8u)); + } + } + for (uint32_t lane = 0u; lane < vector_count; ++lane) _mm256_storeu_ps(destination + lane * 8u, sums[lane]); +} +#endif + + +#if LW_DW_X86 +LW_DW_TARGET +static int depthwise_pair_is_interior(const lw_nhwc_depthwise_desc* desc, + uint32_t output_y, uint32_t output_x) { + int32_t input_y0 = (int32_t)((uint64_t)(output_y + desc->output_row_offset) * desc->stride_h) - + (int32_t)desc->pad_top; + int32_t input_x0 = (int32_t)((uint64_t)output_x * desc->stride_w) - (int32_t)desc->pad_left; + return input_y0 >= 0 && + input_y0 + (int32_t)desc->kernel_h <= (int32_t)desc->input_height && + input_x0 >= 0 && + input_x0 + (int32_t)desc->stride_w + (int32_t)desc->kernel_w <= + (int32_t)desc->input_width; +} + +LW_DW_TARGET +static void depthwise_block32_x2(const float* input, const float* weights, const float* bias, + float* output, const lw_nhwc_depthwise_desc* desc, + uint32_t output_y, uint32_t output_x, uint32_t channel_base) { + __m256 sum0[4]; + __m256 sum1[4]; + int32_t input_y0 = (int32_t)((uint64_t)(output_y + desc->output_row_offset) * desc->stride_h) - + (int32_t)desc->pad_top; + int32_t input_x0 = (int32_t)((uint64_t)output_x * desc->stride_w) - (int32_t)desc->pad_left; + for (uint32_t lane = 0u; lane < 4u; ++lane) { + __m256 initial = bias == NULL ? _mm256_setzero_ps() : + _mm256_loadu_ps(bias + channel_base + lane * 8u); + sum0[lane] = initial; + sum1[lane] = initial; + } + uint32_t tap = 0u; + for (uint32_t ky = 0u; ky < desc->kernel_h; ++ky) { + for (uint32_t kx = 0u; kx < desc->kernel_w; ++kx, ++tap) { + const float* source0 = input + (((size_t)(input_y0 + (int32_t)ky) * desc->input_width + + (uint32_t)(input_x0 + (int32_t)kx)) * desc->channels + channel_base); + const float* source1 = input + (((size_t)(input_y0 + (int32_t)ky) * desc->input_width + + (uint32_t)(input_x0 + (int32_t)desc->stride_w + (int32_t)kx)) * + desc->channels + channel_base); + const float* weight = weights + (size_t)tap * LW_NHWC_DEPTHWISE_BLOCK; + for (uint32_t lane = 0u; lane < 4u; ++lane) { + __m256 w = _mm256_loadu_ps(weight + lane * 8u); + sum0[lane] = _mm256_fmadd_ps(_mm256_loadu_ps(source0 + lane * 8u), w, sum0[lane]); + sum1[lane] = _mm256_fmadd_ps(_mm256_loadu_ps(source1 + lane * 8u), w, sum1[lane]); + } + } + } + float* destination0 = output + (((size_t)output_y * desc->output_width + output_x) * + desc->channels + channel_base); + float* destination1 = destination0 + desc->channels; + if (desc->post_bias != NULL) { + for (uint32_t lane = 0u; lane < 4u; ++lane) { + __m256 post = _mm256_loadu_ps(desc->post_bias + channel_base + lane * 8u); + sum0[lane] = _mm256_add_ps(sum0[lane], post); + sum1[lane] = _mm256_add_ps(sum1[lane], post); + } + } + for (uint32_t lane = 0u; lane < 4u; ++lane) { + _mm256_storeu_ps(destination0 + lane * 8u, sum0[lane]); + _mm256_storeu_ps(destination1 + lane * 8u, sum1[lane]); + } +} +#endif +static lw_status depthwise_execute(const float* input, const float* packed_weights, + const float* bias, float* output, + const lw_nhwc_depthwise_desc* desc, + lw_nhwc_depthwise_stats* stats) { + if (input == NULL || packed_weights == NULL || output == NULL || desc == NULL || + desc->batch == 0u || desc->channels == 0u || + (desc->channels & 7u) != 0u || + desc->input_height == 0u || desc->input_width == 0u || desc->output_height == 0u || + desc->output_width == 0u || desc->kernel_h == 0u || desc->kernel_w == 0u || + desc->stride_h == 0u || desc->stride_w == 0u) return LW_STATUS_INVALID_ARGUMENT; + if ((uint64_t)desc->input_height + desc->pad_top + desc->pad_bottom < desc->kernel_h || + (uint64_t)desc->input_width + desc->pad_left + desc->pad_right < desc->kernel_w || + (uint64_t)desc->output_height + desc->output_row_offset > + (desc->input_height + (uint64_t)desc->pad_top + desc->pad_bottom - desc->kernel_h) / + desc->stride_h + 1u || + ((uint64_t)desc->input_width + desc->pad_left + desc->pad_right - desc->kernel_w) / + desc->stride_w + 1u != desc->output_width) return LW_STATUS_INVALID_SHAPE; +#if LW_DW_X86 + uint32_t taps = desc->kernel_h * desc->kernel_w; + uint32_t blocks = (desc->channels + LW_NHWC_DEPTHWISE_BLOCK - 1u) / LW_NHWC_DEPTHWISE_BLOCK; + for (uint32_t batch = 0u; batch < desc->batch; ++batch) { + const float* input_batch = input + (size_t)batch * desc->input_height * desc->input_width * desc->channels; + float* output_batch = output + (size_t)batch * desc->output_height * desc->output_width * desc->channels; + for (uint32_t output_y = 0u; output_y < desc->output_height; ++output_y) { + for (uint32_t block = 0u; block < blocks; ++block) { + uint32_t channel_base = block * LW_NHWC_DEPTHWISE_BLOCK; + uint32_t remaining = desc->channels - channel_base; + uint32_t current_channels = remaining > LW_NHWC_DEPTHWISE_BLOCK ? LW_NHWC_DEPTHWISE_BLOCK : remaining; + uint32_t vector_count = current_channels / 8u; + const float* block_weights = packed_weights + + (size_t)block * taps * LW_NHWC_DEPTHWISE_BLOCK; + uint32_t output_x = 0u; + while (output_x < desc->output_width) { + if (output_x + 1u < desc->output_width && + vector_count == 4u && + depthwise_pair_is_interior(desc, output_y, output_x)) { + depthwise_block32_x2(input_batch, block_weights, bias, output_batch, desc, + output_y, output_x, channel_base); + if (stats != NULL && stats->x2_invocations != UINT64_MAX) ++stats->x2_invocations; + output_x += 2u; + } else { + depthwise_block32(input_batch, block_weights, bias, output_batch, desc, + output_y, output_x, channel_base, vector_count); + if (stats != NULL && stats->x1_invocations != UINT64_MAX) ++stats->x1_invocations; + ++output_x; + } + } + } + } + } + return LW_STATUS_OK; +#elif LW_DW_WASM128 + /* Preserve the OC32 pack and ascending tap order, but compute four + * adjacent channels per SIMD128 lane group. This path handles borders, + * row shards and channel tails with the same descriptor as the scalar + * implementation. */ + uint32_t taps = desc->kernel_h * desc->kernel_w; + uint32_t blocks = (desc->channels + LW_NHWC_DEPTHWISE_BLOCK - 1u) / + LW_NHWC_DEPTHWISE_BLOCK; + for (uint32_t batch = 0u; batch < desc->batch; ++batch) { + const float* input_batch = input + (size_t)batch * desc->input_height * + desc->input_width * desc->channels; + float* output_batch = output + (size_t)batch * desc->output_height * + desc->output_width * desc->channels; + for (uint32_t oy = 0u; oy < desc->output_height; ++oy) { + int32_t iy0 = (int32_t)((uint64_t)(oy + desc->output_row_offset) * + desc->stride_h) - (int32_t)desc->pad_top; + for (uint32_t ox = 0u; ox < desc->output_width; ++ox) { + int32_t ix0 = (int32_t)((uint64_t)ox * desc->stride_w) - + (int32_t)desc->pad_left; + float* destination = output_batch + + ((size_t)oy * desc->output_width + ox) * desc->channels; + for (uint32_t block = 0u; block < blocks; ++block) { + uint32_t channel_base = block * LW_NHWC_DEPTHWISE_BLOCK; + uint32_t lanes = desc->channels - channel_base; + const float* block_weights = packed_weights + + (size_t)block * taps * LW_NHWC_DEPTHWISE_BLOCK; + v128_t sums[LW_NHWC_DEPTHWISE_BLOCK / 4u]; + if (lanes > LW_NHWC_DEPTHWISE_BLOCK) lanes = LW_NHWC_DEPTHWISE_BLOCK; + for (uint32_t lane = 0u; lane < lanes; lane += 4u) { + sums[lane / 4u] = bias == NULL ? wasm_f32x4_splat(0.0f) : + wasm_v128_load(bias + channel_base + lane); + } + for (uint32_t ky = 0u; ky < desc->kernel_h; ++ky) { + int32_t iy = iy0 + (int32_t)ky; + if (iy < 0 || iy >= (int32_t)desc->input_height) continue; + for (uint32_t kx = 0u; kx < desc->kernel_w; ++kx) { + int32_t ix = ix0 + (int32_t)kx; + if (ix < 0 || ix >= (int32_t)desc->input_width) continue; + const float* source = input_batch + + ((size_t)(uint32_t)iy * desc->input_width + (uint32_t)ix) * + desc->channels + channel_base; + const float* weight = block_weights + + ((size_t)ky * desc->kernel_w + kx) * + LW_NHWC_DEPTHWISE_BLOCK; + for (uint32_t lane = 0u; lane < lanes; lane += 4u) { + sums[lane / 4u] = wasm_f32x4_add(sums[lane / 4u], + wasm_f32x4_mul(wasm_v128_load(source + lane), + wasm_v128_load(weight + lane))); + } + } + } + for (uint32_t lane = 0u; lane < lanes; lane += 4u) { + v128_t sum = sums[lane / 4u]; + if (desc->post_bias != NULL) { + sum = wasm_f32x4_add(sum, + wasm_v128_load(desc->post_bias + channel_base + lane)); + } + wasm_v128_store(destination + channel_base + lane, sum); + } + } + if (stats != NULL && stats->x1_invocations != UINT64_MAX) { + ++stats->x1_invocations; + } + } + } + } + return LW_STATUS_OK; +#else + /* Physical NHWC fallback for non-x86 compiled backends. Preserve the + * packed 32-channel layout and the per-channel tap accumulation order. */ + for (uint32_t batch = 0u; batch < desc->batch; ++batch) { + const float* input_batch = input + (size_t)batch * desc->input_height * + desc->input_width * desc->channels; + float* output_batch = output + (size_t)batch * desc->output_height * + desc->output_width * desc->channels; + for (uint32_t oy = 0u; oy < desc->output_height; ++oy) { + int32_t iy0 = (int32_t)((uint64_t)(oy + desc->output_row_offset) * + desc->stride_h) - (int32_t)desc->pad_top; + for (uint32_t ox = 0u; ox < desc->output_width; ++ox) { + int32_t ix0 = (int32_t)((uint64_t)ox * desc->stride_w) - + (int32_t)desc->pad_left; + for (uint32_t channel = 0u; channel < desc->channels; ++channel) { + uint32_t block = channel / LW_NHWC_DEPTHWISE_BLOCK; + uint32_t lane = channel % LW_NHWC_DEPTHWISE_BLOCK; + float sum = bias == NULL ? 0.0f : bias[channel]; + for (uint32_t ky = 0u; ky < desc->kernel_h; ++ky) { + int32_t iy = iy0 + (int32_t)ky; + if (iy < 0 || iy >= (int32_t)desc->input_height) continue; + for (uint32_t kx = 0u; kx < desc->kernel_w; ++kx) { + int32_t ix = ix0 + (int32_t)kx; + if (ix < 0 || ix >= (int32_t)desc->input_width) continue; + size_t input_index = ((size_t)(uint32_t)iy * desc->input_width + + (uint32_t)ix) * desc->channels + channel; + size_t weight_index = ((size_t)block * desc->kernel_h * + desc->kernel_w + (size_t)ky * desc->kernel_w + kx) * + LW_NHWC_DEPTHWISE_BLOCK + lane; + sum += input_batch[input_index] * packed_weights[weight_index]; + } + } + if (desc->post_bias != NULL) sum += desc->post_bias[channel]; + output_batch[((size_t)oy * desc->output_width + ox) * + desc->channels + channel] = sum; + } + if (stats != NULL && stats->x1_invocations != UINT64_MAX) { + ++stats->x1_invocations; + } + } + } + } + return LW_STATUS_OK; +#endif +} + +lw_status lw_avx2_fma_nhwc_depthwise_f32(const float* input, const float* packed_weights, + const float* bias, float* output, + const lw_nhwc_depthwise_desc* desc) { + return depthwise_execute(input, packed_weights, bias, output, desc, NULL); +} + +lw_status lw_avx2_fma_nhwc_depthwise_profiled_f32(const float* input, const float* packed_weights, + const float* bias, float* output, + const lw_nhwc_depthwise_desc* desc, + lw_nhwc_depthwise_stats* stats) { + return depthwise_execute(input, packed_weights, bias, output, desc, stats); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_pointwise.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_pointwise.c new file mode 100644 index 00000000..78d59258 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_pointwise.c @@ -0,0 +1,816 @@ +#include "nhwc_internal.h" + +#include + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) +#define LW_NHWC_X86 1 +#include +#endif + +static float lw_nhwc_apply_activation(float value, uint16_t activation, + float alpha, float beta) { + (void)alpha; + (void)beta; + if (activation == LW_NHWC_ACT_RELU) { + return value > 0.0f ? value : 0.0f; + } + if (activation == LW_NHWC_ACT_HARDSWISH) { + float gate = value + 3.0f; + if (gate < 0.0f) { + gate = 0.0f; + } else if (gate > 6.0f) { + gate = 6.0f; + } + return value * gate / 6.0f; + } + if (activation == LW_NHWC_ACT_GELU) { + return lw_nhwc_gelu_scalar_exact_f32(value); + } + return value; +} + +static void lw_nhwc_pointwise_scalar(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels) { + const float* bias = epilogue == NULL ? NULL : epilogue->bias; + const float* residual = epilogue == NULL ? NULL : epilogue->residual; + const float* post_bias = epilogue == NULL ? NULL : epilogue->post_bias; + uint16_t activation = epilogue == NULL ? LW_NHWC_ACT_NONE : epilogue->activation; + float alpha = epilogue == NULL ? 0.0f : epilogue->alpha; + float beta = epilogue == NULL ? 0.0f : epilogue->beta; + uint32_t pixel; + + for (pixel = 0u; pixel < pixels; ++pixel) { + uint32_t output_channel; + for (output_channel = 0u; output_channel < output_channels; + ++output_channel) { + uint32_t block = output_channel / LW_NHWC_OC_BLOCK; + uint32_t lane = output_channel % LW_NHWC_OC_BLOCK; + const float* packed = packed_weights + + (size_t)block * input_channels * LW_NHWC_OC_BLOCK; + float sum = bias == NULL ? 0.0f : bias[output_channel]; + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; + ++input_channel) { + sum += input[(size_t)pixel * input_channels + input_channel] * + packed[(size_t)input_channel * LW_NHWC_OC_BLOCK + lane]; + } + if (post_bias != NULL) { + sum += post_bias[output_channel]; + } + if (residual != NULL) { + sum += residual[(size_t)pixel * output_channels + output_channel]; + } + output[(size_t)pixel * output_channels + output_channel] = + lw_nhwc_apply_activation(sum, activation, alpha, beta); + } + } +} + +#if defined(LW_NHWC_X86) +#if defined(__GNUC__) || defined(__clang__) +#define LW_NHWC_AVX2_FMA __attribute__((target("avx2,fma"))) +#else +#define LW_NHWC_AVX2_FMA +#endif + +LW_NHWC_AVX2_FMA +static void lw_nhwc_store_row16(__m256 lo, __m256 hi, + float* output, + const float* residual, + const float* post_bias, + uint16_t activation, + uint32_t channels) { + if (channels < 16u) { + float values[16]; + uint32_t i; + _mm256_storeu_ps(values, lo); + _mm256_storeu_ps(values + 8u, hi); + for (i = 0u; i < channels; ++i) { + float value = values[i]; + if (post_bias != NULL) value += post_bias[i]; + if (residual != NULL) value += residual[i]; + values[i] = value; + } + if (activation == LW_NHWC_ACT_GELU) { + /* Match the standalone pass layout exactly: full 8-lane groups go + * through the vector polynomial, the sub-8 tail through the + * scalar erff helper. channels < 16 here, so at most one full + * vector group exists. */ + uint32_t vector_end = channels & ~7u; + if (vector_end >= 8u) { + _mm256_storeu_ps(values, + lw_nhwc_avx2_gelu_vector_exact_f32(_mm256_loadu_ps(values))); + } + for (i = vector_end; i < channels; ++i) { + values[i] = lw_nhwc_gelu_scalar_exact_f32(values[i]); + } + for (i = 0u; i < channels; ++i) output[i] = values[i]; + return; + } + for (i = 0u; i < channels; ++i) { + float value = values[i]; + if (activation == LW_NHWC_ACT_RELU) { + value = value < 0.0f ? 0.0f : value; + } else if (activation == LW_NHWC_ACT_HARDSWISH) { + float gate = value + 3.0f; + if (gate < 0.0f) gate = 0.0f; + if (gate > 6.0f) gate = 6.0f; + value = value * gate * (1.0f / 6.0f); + } + output[i] = value; + } + return; + } + if (post_bias != NULL) { + lo = _mm256_add_ps(lo, _mm256_loadu_ps(post_bias)); + hi = _mm256_add_ps(hi, _mm256_loadu_ps(post_bias + 8u)); + } + if (residual != NULL) { + lo = _mm256_add_ps(lo, _mm256_loadu_ps(residual)); + hi = _mm256_add_ps(hi, _mm256_loadu_ps(residual + 8u)); + } + if (activation == LW_NHWC_ACT_RELU) { + __m256 zero = _mm256_setzero_ps(); + lo = _mm256_max_ps(lo, zero); + hi = _mm256_max_ps(hi, zero); + } else if (activation == LW_NHWC_ACT_HARDSWISH) { + __m256 three = _mm256_set1_ps(3.0f); + __m256 six = _mm256_set1_ps(6.0f); + __m256 inv_six = _mm256_set1_ps(1.0f / 6.0f); + __m256 zero = _mm256_setzero_ps(); + __m256 gate_lo = _mm256_min_ps(_mm256_max_ps(_mm256_add_ps(lo, three), zero), six); + __m256 gate_hi = _mm256_min_ps(_mm256_max_ps(_mm256_add_ps(hi, three), zero), six); + lo = _mm256_mul_ps(_mm256_mul_ps(lo, gate_lo), inv_six); + hi = _mm256_mul_ps(_mm256_mul_ps(hi, gate_hi), inv_six); + } else if (activation == LW_NHWC_ACT_GELU) { + lo = lw_nhwc_avx2_gelu_vector_exact_f32(lo); + hi = lw_nhwc_avx2_gelu_vector_exact_f32(hi); + } + _mm256_storeu_ps(output, lo); + _mm256_storeu_ps(output + 8u, hi); +} + +LW_NHWC_AVX2_FMA +static void lw_nhwc_tile_rows16(const float* input, + const float* packed_weights, + const float* bias, + const float* residual, + const float* post_bias, + float* output, + uint32_t rows, + uint32_t input_channels, + uint32_t output_stride, + uint16_t activation, + uint32_t output_channels) { + __m256 lo[6]; + __m256 hi[6]; + uint32_t row; + uint32_t input_channel; + __m256 zero = _mm256_setzero_ps(); + + for (row = 0u; row < 6u; ++row) { + if (bias != NULL && output_channels < LW_NHWC_OC_BLOCK) { + float bias_values[LW_NHWC_OC_BLOCK] = {0.0f}; + for (uint32_t lane = 0u; lane < output_channels; ++lane) { + bias_values[lane] = bias[lane]; + } + lo[row] = _mm256_loadu_ps(bias_values); + hi[row] = _mm256_loadu_ps(bias_values + 8u); + } else if (bias != NULL) { + lo[row] = _mm256_loadu_ps(bias); + hi[row] = _mm256_loadu_ps(bias + 8u); + } else { + lo[row] = zero; + hi[row] = zero; + } + } + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* weights = packed_weights + (size_t)input_channel * 16u; + __m256 w0 = _mm256_loadu_ps(weights); + __m256 w1 = _mm256_loadu_ps(weights + 8u); + for (row = 0u; row < 6u; ++row) { + const float* row_input = input + (size_t)(row < rows ? row : 0u) * input_channels; + __m256 value = _mm256_set1_ps(row_input[input_channel]); + lo[row] = _mm256_fmadd_ps(value, w0, lo[row]); + hi[row] = _mm256_fmadd_ps(value, w1, hi[row]); + } + } + for (row = 0u; row < rows; ++row) { + float* row_output = output + (size_t)row * output_stride; + const float* row_residual = residual == NULL ? NULL : + residual + (size_t)row * output_stride; + lw_nhwc_store_row16(lo[row], hi[row], row_output, row_residual, + post_bias, activation, output_channels); + } +} + +/* Grouped-path worker: one noinline call covers a whole pixel range of a + * single full 16-channel output block, paying the call and stack-frame + * prologue once per (pixel group, channel block) instead of once per 6-row + * tile. Bit-identical to repeated lw_nhwc_tile_rows16 calls: same bias + * initialisation, same ascending input-channel FMA order, same store_row16 + * epilogue. Tail tiles (rows < 6) delegate to lw_nhwc_tile_rows16. */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((noinline)) +#elif defined(_MSC_VER) +__declspec(noinline) +#endif +LW_NHWC_AVX2_FMA +static void lw_nhwc_ocblock16_tile_range(const float* input, + const float* packed_weights, + const float* bias, + const float* residual, + const float* post_bias, + float* output, + uint32_t pixel_begin, + uint32_t pixel_end, + uint32_t input_channels, + uint32_t output_stride, + uint16_t activation) { + uint32_t pixel; + for (pixel = pixel_begin; pixel < pixel_end; pixel += LW_NHWC_PIXEL_TILE) { + uint32_t rows = pixel_end - pixel; + const float* tile_input = input + (size_t)pixel * input_channels; + float* tile_output = output + (size_t)pixel * output_stride; + const float* tile_residual = residual == NULL ? NULL : + residual + (size_t)pixel * output_stride; + __m256 lo0, lo1, lo2, lo3, lo4, lo5; + __m256 hi0, hi1, hi2, hi3, hi4, hi5; + uint32_t input_channel; + if (rows > LW_NHWC_PIXEL_TILE) { + rows = LW_NHWC_PIXEL_TILE; + } + if (rows < LW_NHWC_PIXEL_TILE) { + lw_nhwc_tile_rows16(tile_input, packed_weights, bias, tile_residual, + post_bias, tile_output, rows, input_channels, + output_stride, activation, LW_NHWC_OC_BLOCK); + continue; + } + if (bias != NULL) { + __m256 bias_lo = _mm256_loadu_ps(bias); + __m256 bias_hi = _mm256_loadu_ps(bias + 8u); + lo0 = bias_lo; lo1 = bias_lo; lo2 = bias_lo; + lo3 = bias_lo; lo4 = bias_lo; lo5 = bias_lo; + hi0 = bias_hi; hi1 = bias_hi; hi2 = bias_hi; + hi3 = bias_hi; hi4 = bias_hi; hi5 = bias_hi; + } else { + __m256 zero = _mm256_setzero_ps(); + lo0 = zero; lo1 = zero; lo2 = zero; + lo3 = zero; lo4 = zero; lo5 = zero; + hi0 = zero; hi1 = zero; hi2 = zero; + hi3 = zero; hi4 = zero; hi5 = zero; + } + for (input_channel = 0u; input_channel < input_channels; + ++input_channel) { + const float* weights = packed_weights + (size_t)input_channel * 16u; + __m256 w0 = _mm256_loadu_ps(weights); + __m256 w1 = _mm256_loadu_ps(weights + 8u); + __m256 value; + value = _mm256_set1_ps(tile_input[input_channel]); + lo0 = _mm256_fmadd_ps(value, w0, lo0); + hi0 = _mm256_fmadd_ps(value, w1, hi0); + value = _mm256_set1_ps( + tile_input[(size_t)input_channels + input_channel]); + lo1 = _mm256_fmadd_ps(value, w0, lo1); + hi1 = _mm256_fmadd_ps(value, w1, hi1); + value = _mm256_set1_ps( + tile_input[(size_t)2u * input_channels + input_channel]); + lo2 = _mm256_fmadd_ps(value, w0, lo2); + hi2 = _mm256_fmadd_ps(value, w1, hi2); + value = _mm256_set1_ps( + tile_input[(size_t)3u * input_channels + input_channel]); + lo3 = _mm256_fmadd_ps(value, w0, lo3); + hi3 = _mm256_fmadd_ps(value, w1, hi3); + value = _mm256_set1_ps( + tile_input[(size_t)4u * input_channels + input_channel]); + lo4 = _mm256_fmadd_ps(value, w0, lo4); + hi4 = _mm256_fmadd_ps(value, w1, hi4); + value = _mm256_set1_ps( + tile_input[(size_t)5u * input_channels + input_channel]); + lo5 = _mm256_fmadd_ps(value, w0, lo5); + hi5 = _mm256_fmadd_ps(value, w1, hi5); + } + lw_nhwc_store_row16(lo0, hi0, tile_output, tile_residual, post_bias, + activation, LW_NHWC_OC_BLOCK); + lw_nhwc_store_row16(lo1, hi1, tile_output + output_stride, + tile_residual == NULL ? NULL : + tile_residual + output_stride, + post_bias, activation, LW_NHWC_OC_BLOCK); + lw_nhwc_store_row16(lo2, hi2, tile_output + (size_t)2u * output_stride, + tile_residual == NULL ? NULL : + tile_residual + (size_t)2u * output_stride, + post_bias, activation, LW_NHWC_OC_BLOCK); + lw_nhwc_store_row16(lo3, hi3, tile_output + (size_t)3u * output_stride, + tile_residual == NULL ? NULL : + tile_residual + (size_t)3u * output_stride, + post_bias, activation, LW_NHWC_OC_BLOCK); + lw_nhwc_store_row16(lo4, hi4, tile_output + (size_t)4u * output_stride, + tile_residual == NULL ? NULL : + tile_residual + (size_t)4u * output_stride, + post_bias, activation, LW_NHWC_OC_BLOCK); + lw_nhwc_store_row16(lo5, hi5, tile_output + (size_t)5u * output_stride, + tile_residual == NULL ? NULL : + tile_residual + (size_t)5u * output_stride, + post_bias, activation, LW_NHWC_OC_BLOCK); + } +} + +/* Whole-tensor fast path: one call per conv, tile-outer/oc-inner loops with + * the unrolled 6x16 accumulate inline. Per-tile function-call overhead of the + * grouped path disappears. Bit-identical to lw_nhwc_tile_rows16: same bias + * initialisation, same ascending input-channel FMA order, same store_row16 + * epilogue. Tail tiles (rows < 6) delegate to lw_nhwc_tile_rows16. */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((noinline)) +#elif defined(_MSC_VER) +__declspec(noinline) +#endif +LW_NHWC_AVX2_FMA +static void lw_nhwc_pointwise_unrolled_all(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels) { + const float* bias = epilogue->bias; + const float* residual = epilogue->residual; + const float* post_bias = epilogue->post_bias; + uint16_t activation = epilogue->activation; + uint32_t tile; + for (tile = 0u; tile < pixels; tile += LW_NHWC_PIXEL_TILE) { + uint32_t rows = pixels - tile; + uint32_t output_channel; + if (rows > LW_NHWC_PIXEL_TILE) rows = LW_NHWC_PIXEL_TILE; + for (output_channel = 0u; output_channel < output_channels; + output_channel += LW_NHWC_OC_BLOCK) { + const float* packed = packed_weights + + (size_t)(output_channel / LW_NHWC_OC_BLOCK) * input_channels * + LW_NHWC_OC_BLOCK; + const float* tile_input = input + (size_t)tile * input_channels; + float* tile_output = output + (size_t)tile * output_channels + + output_channel; + const float* tile_residual = residual == NULL ? NULL : + residual + (size_t)tile * output_channels + output_channel; + const float* block_bias = bias == NULL ? NULL : bias + output_channel; + const float* block_post_bias = + post_bias == NULL ? NULL : post_bias + output_channel; + if (rows < LW_NHWC_PIXEL_TILE) { + lw_nhwc_tile_rows16(tile_input, packed, block_bias, tile_residual, + block_post_bias, tile_output, rows, + input_channels, output_channels, activation, + LW_NHWC_OC_BLOCK); + continue; + } + { + __m256 lo0, lo1, lo2, lo3, lo4, lo5; + __m256 hi0, hi1, hi2, hi3, hi4, hi5; + uint32_t input_channel; + if (block_bias != NULL) { + __m256 bias_lo = _mm256_loadu_ps(block_bias); + __m256 bias_hi = _mm256_loadu_ps(block_bias + 8u); + lo0 = bias_lo; lo1 = bias_lo; lo2 = bias_lo; + lo3 = bias_lo; lo4 = bias_lo; lo5 = bias_lo; + hi0 = bias_hi; hi1 = bias_hi; hi2 = bias_hi; + hi3 = bias_hi; hi4 = bias_hi; hi5 = bias_hi; + } else { + __m256 zero = _mm256_setzero_ps(); + lo0 = zero; lo1 = zero; lo2 = zero; + lo3 = zero; lo4 = zero; lo5 = zero; + hi0 = zero; hi1 = zero; hi2 = zero; + hi3 = zero; hi4 = zero; hi5 = zero; + } + for (input_channel = 0u; input_channel < input_channels; + ++input_channel) { + const float* weights = + packed + (size_t)input_channel * 16u; + __m256 w0 = _mm256_loadu_ps(weights); + __m256 w1 = _mm256_loadu_ps(weights + 8u); + __m256 value; + value = _mm256_set1_ps(tile_input[input_channel]); + lo0 = _mm256_fmadd_ps(value, w0, lo0); + hi0 = _mm256_fmadd_ps(value, w1, hi0); + value = _mm256_set1_ps( + tile_input[(size_t)input_channels + input_channel]); + lo1 = _mm256_fmadd_ps(value, w0, lo1); + hi1 = _mm256_fmadd_ps(value, w1, hi1); + value = _mm256_set1_ps( + tile_input[(size_t)2u * input_channels + input_channel]); + lo2 = _mm256_fmadd_ps(value, w0, lo2); + hi2 = _mm256_fmadd_ps(value, w1, hi2); + value = _mm256_set1_ps( + tile_input[(size_t)3u * input_channels + input_channel]); + lo3 = _mm256_fmadd_ps(value, w0, lo3); + hi3 = _mm256_fmadd_ps(value, w1, hi3); + value = _mm256_set1_ps( + tile_input[(size_t)4u * input_channels + input_channel]); + lo4 = _mm256_fmadd_ps(value, w0, lo4); + hi4 = _mm256_fmadd_ps(value, w1, hi4); + value = _mm256_set1_ps( + tile_input[(size_t)5u * input_channels + input_channel]); + lo5 = _mm256_fmadd_ps(value, w0, lo5); + hi5 = _mm256_fmadd_ps(value, w1, hi5); + } + lw_nhwc_store_row16(lo0, hi0, tile_output, tile_residual, + block_post_bias, activation, LW_NHWC_OC_BLOCK); + lw_nhwc_store_row16( + lo1, hi1, tile_output + output_channels, + tile_residual == NULL ? NULL : tile_residual + output_channels, + block_post_bias, activation, LW_NHWC_OC_BLOCK); + lw_nhwc_store_row16( + lo2, hi2, tile_output + (size_t)2u * output_channels, + tile_residual == NULL ? NULL : + tile_residual + (size_t)2u * output_channels, + block_post_bias, activation, LW_NHWC_OC_BLOCK); + lw_nhwc_store_row16( + lo3, hi3, tile_output + (size_t)3u * output_channels, + tile_residual == NULL ? NULL : + tile_residual + (size_t)3u * output_channels, + block_post_bias, activation, LW_NHWC_OC_BLOCK); + lw_nhwc_store_row16( + lo4, hi4, tile_output + (size_t)4u * output_channels, + tile_residual == NULL ? NULL : + tile_residual + (size_t)4u * output_channels, + block_post_bias, activation, LW_NHWC_OC_BLOCK); + lw_nhwc_store_row16( + lo5, hi5, tile_output + (size_t)5u * output_channels, + tile_residual == NULL ? NULL : + tile_residual + (size_t)5u * output_channels, + block_post_bias, activation, LW_NHWC_OC_BLOCK); + } + } + } +} +#endif + +void lw_avx2_fma_nhwc_pointwise_grouped_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels, + uint32_t group_tiles) { +#if defined(LW_NHWC_X86) + const float* bias = epilogue == NULL ? NULL : epilogue->bias; + const float* residual = epilogue == NULL ? NULL : epilogue->residual; + const float* post_bias = epilogue == NULL ? NULL : epilogue->post_bias; +#endif + uint16_t activation = epilogue == NULL ? LW_NHWC_ACT_NONE : epilogue->activation; + + if (input == NULL || packed_weights == NULL || output == NULL || + input_channels == 0u || output_channels == 0u || pixels == 0u) { + return; + } + if (activation != LW_NHWC_ACT_NONE && activation != LW_NHWC_ACT_RELU && + activation != LW_NHWC_ACT_HARDSWISH && + activation != LW_NHWC_ACT_GELU) { + lw_nhwc_pointwise_scalar(input, packed_weights, epilogue, output, pixels, + input_channels, output_channels); + return; + } + +#if defined(LW_NHWC_X86) + { + uint32_t tile_count = (pixels + LW_NHWC_PIXEL_TILE - 1u) / + LW_NHWC_PIXEL_TILE; + uint32_t tile_group = group_tiles == 0u ? tile_count : group_tiles; + uint32_t group_begin; + if (tile_group == 0u) { + tile_group = 1u; + } + for (group_begin = 0u; group_begin < tile_count; + group_begin += tile_group) { + uint32_t group_end = group_begin + tile_group; + uint32_t output_channel; + if (group_end > tile_count) { + group_end = tile_count; + } + for (output_channel = 0u; output_channel < output_channels; + output_channel += LW_NHWC_OC_BLOCK) { + const float* packed = packed_weights + + (size_t)(output_channel / LW_NHWC_OC_BLOCK) * input_channels * + LW_NHWC_OC_BLOCK; + uint32_t block_channels = output_channels - output_channel; + if (block_channels > LW_NHWC_OC_BLOCK) { + block_channels = LW_NHWC_OC_BLOCK; + } + if (block_channels == LW_NHWC_OC_BLOCK) { + /* Full channel block: one worker call covers the whole + * pixel group, paying the stack-frame prologue once. */ + uint32_t pixel_begin = group_begin * LW_NHWC_PIXEL_TILE; + uint32_t pixel_end = group_end * LW_NHWC_PIXEL_TILE; + if (pixel_end > pixels) { + pixel_end = pixels; + } + lw_nhwc_ocblock16_tile_range( + input, packed, + bias == NULL ? NULL : bias + output_channel, + residual == NULL ? NULL : residual + output_channel, + post_bias == NULL ? NULL : post_bias + output_channel, + output + output_channel, + pixel_begin, pixel_end, + input_channels, output_channels, activation); + continue; + } + for (uint32_t tile = group_begin; tile < group_end; ++tile) { + uint32_t pixel = tile * LW_NHWC_PIXEL_TILE; + uint32_t rows = pixels - pixel; + const float* tile_input = input + (size_t)pixel * input_channels; + float* tile_output = output + (size_t)pixel * output_channels + + output_channel; + const float* tile_residual = residual == NULL ? NULL : + residual + (size_t)pixel * output_channels + output_channel; + if (rows > LW_NHWC_PIXEL_TILE) { + rows = LW_NHWC_PIXEL_TILE; + } + lw_nhwc_tile_rows16(tile_input, packed, + bias == NULL ? NULL : bias + output_channel, + tile_residual, + post_bias == NULL ? NULL : + post_bias + output_channel, + tile_output, rows, + input_channels, output_channels, activation, + block_channels); + } + } + } + } +#else + (void)group_tiles; + lw_nhwc_pointwise_scalar(input, packed_weights, epilogue, output, pixels, + input_channels, output_channels); +#endif +} + + +#if defined(LW_NHWC_X86) +LW_NHWC_AVX2_FMA +static void lw_nhwc_tile_rows16_4(const float* input, const float* packed_weights, + const float* bias, const float* residual, + const float* post_bias, float* output, + uint32_t rows, uint32_t input_channels, + uint32_t output_stride, uint16_t activation) { + __m256 lo[4], hi[4]; + __m256 zero = _mm256_setzero_ps(); + uint32_t row; + for (row = 0u; row < 4u; ++row) { + if (bias != NULL) { + lo[row] = _mm256_loadu_ps(bias); + hi[row] = _mm256_loadu_ps(bias + 8u); + } else { + lo[row] = zero; + hi[row] = zero; + } + } + for (uint32_t input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* weights = packed_weights + (size_t)input_channel * 16u; + __m256 w0 = _mm256_loadu_ps(weights); + __m256 w1 = _mm256_loadu_ps(weights + 8u); + for (row = 0u; row < rows; ++row) { + __m256 value = _mm256_set1_ps(input[(size_t)row * input_channels + input_channel]); + lo[row] = _mm256_fmadd_ps(value, w0, lo[row]); + hi[row] = _mm256_fmadd_ps(value, w1, hi[row]); + } + } + for (row = 0u; row < rows; ++row) { + float* row_output = output + (size_t)row * output_stride; + const float* row_residual = residual == NULL ? NULL : residual + (size_t)row * output_stride; + lw_nhwc_store_row16(lo[row], hi[row], row_output, row_residual, + post_bias, activation, LW_NHWC_OC_BLOCK); + } +} + +LW_NHWC_AVX2_FMA +static __m256 lw_nhwc_apply_vec(__m256 value, uint16_t activation) { + if (activation == LW_NHWC_ACT_RELU) { + return _mm256_max_ps(value, _mm256_setzero_ps()); + } + if (activation == LW_NHWC_ACT_HARDSWISH) { + __m256 gate = _mm256_min_ps(_mm256_max_ps(_mm256_add_ps(value, _mm256_set1_ps(3.0f)), + _mm256_setzero_ps()), + _mm256_set1_ps(6.0f)); + return _mm256_mul_ps(_mm256_mul_ps(value, gate), _mm256_set1_ps(1.0f / 6.0f)); + } + if (activation == LW_NHWC_ACT_GELU) { + return lw_nhwc_avx2_gelu_vector_exact_f32(value); + } + return value; +} + +LW_NHWC_AVX2_FMA +static void lw_nhwc_store_row32(__m256* accumulators, float* output, + const float* residual, const float* post_bias, + uint16_t activation) { + for (uint32_t vector = 0u; vector < 4u; ++vector) { + __m256 value = accumulators[vector]; + if (post_bias != NULL) value = _mm256_add_ps(value, _mm256_loadu_ps(post_bias + vector * 8u)); + if (residual != NULL) value = _mm256_add_ps(value, _mm256_loadu_ps(residual + vector * 8u)); + value = lw_nhwc_apply_vec(value, activation); + _mm256_storeu_ps(output + vector * 8u, value); + } +} + +LW_NHWC_AVX2_FMA +static void lw_nhwc_tile_rows32(const float* input, const float* packed_weights, + const float* bias, const float* residual, + const float* post_bias, float* output, + uint32_t rows, uint32_t input_channels, + uint32_t output_stride, uint16_t activation, + uint32_t row_tile) { + __m256 accumulators[3][4]; + __m256 zero = _mm256_setzero_ps(); + uint32_t row; + for (row = 0u; row < row_tile; ++row) { + for (uint32_t vector = 0u; vector < 4u; ++vector) { + uint32_t lane_offset = vector * 8u; + accumulators[row][vector] = bias == NULL ? zero : _mm256_loadu_ps(bias + lane_offset); + } + } + for (uint32_t input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* weights0 = packed_weights + (size_t)input_channel * 16u; + const float* weights1 = weights0 + (size_t)input_channels * 16u; + __m256 weights[4] = { + _mm256_loadu_ps(weights0), _mm256_loadu_ps(weights0 + 8u), + _mm256_loadu_ps(weights1), _mm256_loadu_ps(weights1 + 8u) + }; + for (row = 0u; row < rows; ++row) { + __m256 value = _mm256_set1_ps(input[(size_t)row * input_channels + input_channel]); + for (uint32_t vector = 0u; vector < 4u; ++vector) { + accumulators[row][vector] = + _mm256_fmadd_ps(value, weights[vector], accumulators[row][vector]); + } + } + } + for (row = 0u; row < rows; ++row) { + float* row_output = output + (size_t)row * output_stride; + const float* row_residual = residual == NULL ? NULL : residual + (size_t)row * output_stride; + lw_nhwc_store_row32(accumulators[row], row_output, row_residual, + post_bias, activation); + } +} +#endif + +void lw_avx2_fma_nhwc_pointwise_4x16_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels) { + if (output_channels % LW_NHWC_OC_BLOCK != 0u || + epilogue == NULL || + (epilogue->activation != LW_NHWC_ACT_NONE && + epilogue->activation != LW_NHWC_ACT_RELU && + epilogue->activation != LW_NHWC_ACT_HARDSWISH && + epilogue->activation != LW_NHWC_ACT_GELU)) { + lw_avx2_fma_nhwc_pointwise_f32(input, packed_weights, epilogue, output, + pixels, input_channels, output_channels); + return; + } +#if defined(LW_NHWC_X86) + for (uint32_t tile = 0u; tile < pixels; tile += 4u) { + uint32_t rows = pixels - tile; + if (rows > 4u) rows = 4u; + for (uint32_t output_channel = 0u; output_channel < output_channels; + output_channel += LW_NHWC_OC_BLOCK) { + const float* packed = packed_weights + + (size_t)(output_channel / LW_NHWC_OC_BLOCK) * input_channels * LW_NHWC_OC_BLOCK; + lw_nhwc_tile_rows16_4(input + (size_t)tile * input_channels, packed, + epilogue->bias == NULL ? NULL : epilogue->bias + output_channel, + epilogue->residual == NULL ? NULL : + epilogue->residual + (size_t)tile * output_channels + output_channel, + epilogue->post_bias == NULL ? NULL : + epilogue->post_bias + output_channel, + output + (size_t)tile * output_channels + output_channel, + rows, input_channels, output_channels, epilogue->activation); + } + } +#else + lw_avx2_fma_nhwc_pointwise_f32(input, packed_weights, epilogue, output, + pixels, input_channels, output_channels); +#endif +} + +static void lw_avx2_fma_nhwc_pointwise_32x_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels, + uint32_t row_tile) { + if (output_channels % 32u != 0u || + epilogue == NULL || + (epilogue->activation != LW_NHWC_ACT_NONE && + epilogue->activation != LW_NHWC_ACT_RELU && + epilogue->activation != LW_NHWC_ACT_HARDSWISH && + epilogue->activation != LW_NHWC_ACT_GELU)) { + lw_avx2_fma_nhwc_pointwise_f32(input, packed_weights, epilogue, output, + pixels, input_channels, output_channels); + return; + } +#if defined(LW_NHWC_X86) + for (uint32_t tile = 0u; tile < pixels; tile += row_tile) { + uint32_t rows = pixels - tile; + if (rows > row_tile) rows = row_tile; + for (uint32_t output_channel = 0u; output_channel < output_channels; + output_channel += 32u) { + const float* packed = packed_weights + + (size_t)(output_channel / LW_NHWC_OC_BLOCK) * input_channels * LW_NHWC_OC_BLOCK; + lw_nhwc_tile_rows32(input + (size_t)tile * input_channels, packed, + epilogue->bias == NULL ? NULL : epilogue->bias + output_channel, + epilogue->residual == NULL ? NULL : + epilogue->residual + (size_t)tile * output_channels + output_channel, + epilogue->post_bias == NULL ? NULL : + epilogue->post_bias + output_channel, + output + (size_t)tile * output_channels + output_channel, + rows, input_channels, output_channels, epilogue->activation, row_tile); + } + } +#else + (void)row_tile; + lw_avx2_fma_nhwc_pointwise_f32(input, packed_weights, epilogue, output, + pixels, input_channels, output_channels); +#endif +} + +void lw_avx2_fma_nhwc_pointwise_3x32_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels) { + lw_avx2_fma_nhwc_pointwise_32x_f32(input, packed_weights, epilogue, output, + pixels, input_channels, output_channels, 3u); +} + +void lw_avx2_fma_nhwc_pointwise_2x32_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels) { + lw_avx2_fma_nhwc_pointwise_32x_f32(input, packed_weights, epilogue, output, + pixels, input_channels, output_channels, 2u); +} + +void lw_avx2_fma_nhwc_pointwise_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, + uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels) { +#if defined(LW_NHWC_X86) + /* Whole-tensor fast path: tile loop + unrolled 6x16 accumulate live in one + * function, so the per-tile call/prologue overhead of the grouped path + * disappears. Only for weights that fit L2 comfortably (tile-outer order + * re-streams the weights once per 6-row tile) and full 16-channel blocks. + * Everything else keeps the grouped order, which is weight-cache friendly + * for large channel counts. */ + if (output_channels % LW_NHWC_OC_BLOCK == 0u && + (uint64_t)input_channels * output_channels * sizeof(float) <= + UINT64_C(262144) && + epilogue != NULL && + (epilogue->activation == LW_NHWC_ACT_NONE || + epilogue->activation == LW_NHWC_ACT_RELU || + epilogue->activation == LW_NHWC_ACT_HARDSWISH || + epilogue->activation == LW_NHWC_ACT_GELU)) { + lw_nhwc_pointwise_unrolled_all(input, packed_weights, epilogue, output, + pixels, input_channels, output_channels); + return; + } +#endif + { + /* Adaptive pixel grouping: make each pixel group as large as possible + * while its input slice stays L2-resident (~448 KB budget, leaving + * room for the output slice and weight block), so the per-oc-block + * input re-reads hit L2 and the packed weights stream from DRAM as + * few times as possible. Small inputs collapse to a single group and + * the weights stream exactly once; huge detection feature maps still + * get bounded input slices. Bit-identical: grouping only reorders + * independent (tile, oc block) computations. */ + uint32_t tile_count = + (pixels + LW_NHWC_PIXEL_TILE - 1u) / LW_NHWC_PIXEL_TILE; + uint64_t row_bytes = (uint64_t)input_channels * sizeof(float); + uint32_t budget_pixels = + (uint32_t)(UINT64_C(458752) / (row_bytes == 0u ? 1u : row_bytes)); + uint32_t group_tiles = + (budget_pixels + LW_NHWC_PIXEL_TILE - 1u) / LW_NHWC_PIXEL_TILE; + if (group_tiles < LW_NHWC_POINTWISE_GROUP_TILES) { + group_tiles = LW_NHWC_POINTWISE_GROUP_TILES; + } + if (group_tiles > tile_count) { + group_tiles = tile_count; + } + lw_avx2_fma_nhwc_pointwise_grouped_f32( + input, packed_weights, epilogue, output, pixels, input_channels, + output_channels, group_tiles); + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_support.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_support.c new file mode 100644 index 00000000..f31edfed --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_nhwc_support.c @@ -0,0 +1,302 @@ +#include "nhwc_internal.h" + +#include +#include +#include +#include + +#if defined(_M_X64) || defined(__x86_64__) || defined(_M_IX86) || defined(__i386__) +#include +#define LW_NHWC_SUPPORT_X86 1 +#else +#define LW_NHWC_SUPPORT_X86 0 +#endif + +#if LW_NHWC_SUPPORT_X86 && (defined(__GNUC__) || defined(__clang__)) +#define LW_NHWC_SUPPORT_TARGET __attribute__((target("avx2,fma"))) +#else +#define LW_NHWC_SUPPORT_TARGET +#endif + +LW_NHWC_SUPPORT_TARGET +void lw_avx2_nhwc_affine_f32(const float* input, const float* mul, const float* add, + float* output, uint32_t pixels, uint32_t channels) { + uint32_t pixel; + if (input == NULL || mul == NULL || add == NULL || output == NULL) return; + for (pixel = 0u; pixel < pixels; ++pixel) { + uint32_t channel = 0u; +#if LW_NHWC_SUPPORT_X86 + for (; channel + 8u <= channels; channel += 8u) { + __m256 x = _mm256_loadu_ps(input + (size_t)pixel * channels + channel); + __m256 scale = _mm256_loadu_ps(mul + channel); + __m256 bias = _mm256_loadu_ps(add + channel); + _mm256_storeu_ps(output + (size_t)pixel * channels + channel, + _mm256_fmadd_ps(x, scale, bias)); + } +#endif + for (; channel < channels; ++channel) { + size_t index = (size_t)pixel * channels + channel; + output[index] = input[index] * mul[channel] + add[channel]; + } + } +} + + +LW_NHWC_SUPPORT_TARGET +void lw_avx2_nchw_affine_f32(const float* input, const float* mul, const float* add, + float* output, uint32_t channels, uint32_t spatial) { + if (input == NULL || mul == NULL || add == NULL || output == NULL) return; + for (uint32_t c = 0u; c < channels; ++c) { + const float* source = input + (size_t)c * spatial; + float* destination = output + (size_t)c * spatial; + uint32_t i = 0u; +#if LW_NHWC_SUPPORT_X86 + const __m256 scale = _mm256_set1_ps(mul[c]); + const __m256 bias = _mm256_set1_ps(add[c]); + for (; i + 8u <= spatial; i += 8u) { + const __m256 x = _mm256_loadu_ps(source + i); + _mm256_storeu_ps(destination + i, _mm256_fmadd_ps(x, scale, bias)); + } +#endif + for (; i < spatial; ++i) destination[i] = source[i] * mul[c] + add[c]; + } +} + +LW_NHWC_SUPPORT_TARGET +void lw_avx2_nhwc_reduce_mean_hw_f32(const float* input, float* output, + uint32_t batch, uint32_t height, + uint32_t width, uint32_t channels) { + uint32_t n; + if (input == NULL || output == NULL || batch == 0u || height == 0u || + width == 0u || channels == 0u) return; + for (n = 0u; n < batch; ++n) { + uint32_t channel = 0u; +#if LW_NHWC_SUPPORT_X86 + for (; channel + 8u <= channels; channel += 8u) { + __m256 sum = _mm256_setzero_ps(); + for (uint32_t y = 0u; y < height; ++y) { + for (uint32_t x = 0u; x < width; ++x) { + const float* source = input + ((((size_t)n * height + y) * width + x) * channels + channel); + sum = _mm256_add_ps(sum, _mm256_loadu_ps(source)); + } + } + sum = _mm256_div_ps(sum, _mm256_set1_ps((float)height * (float)width)); + _mm256_storeu_ps(output + (size_t)n * channels + channel, sum); + } +#endif + for (; channel < channels; ++channel) { + float sum = 0.0f; + for (uint32_t y = 0u; y < height; ++y) { + for (uint32_t x = 0u; x < width; ++x) { + sum += input[(((size_t)n * height + y) * width + x) * channels + channel]; + } + } + output[(size_t)n * channels + channel] = sum / ((float)height * (float)width); + } + } +} + +static void lw_nhwc_pool_valid_window(int32_t input_y0, int32_t input_x0, + uint32_t input_height, uint32_t input_width, + uint32_t kernel_h, uint32_t kernel_w, + uint32_t* ky_begin, uint32_t* ky_end, + uint32_t* kx_begin, uint32_t* kx_end) { + int32_t y_end = (int32_t)kernel_h; + int32_t x_end = (int32_t)kernel_w; + int32_t y_begin = input_y0 < 0 ? -input_y0 : 0; + int32_t x_begin = input_x0 < 0 ? -input_x0 : 0; + if (input_y0 + y_end > (int32_t)input_height) y_end = (int32_t)input_height - input_y0; + if (input_x0 + x_end > (int32_t)input_width) x_end = (int32_t)input_width - input_x0; + if (y_begin < 0) y_begin = 0; + if (x_begin < 0) x_begin = 0; + if (y_end < y_begin) y_end = y_begin; + if (x_end < x_begin) x_end = x_begin; + if (y_begin > (int32_t)kernel_h) y_begin = (int32_t)kernel_h; + if (x_begin > (int32_t)kernel_w) x_begin = (int32_t)kernel_w; + if (y_end > (int32_t)kernel_h) y_end = (int32_t)kernel_h; + if (x_end > (int32_t)kernel_w) x_end = (int32_t)kernel_w; + *ky_begin = (uint32_t)y_begin; + *ky_end = (uint32_t)y_end; + *kx_begin = (uint32_t)x_begin; + *kx_end = (uint32_t)x_end; +} + +LW_NHWC_SUPPORT_TARGET +void lw_avx2_nhwc_reduce_mean_hw_strided_f32(const float* input, float* output, + uint32_t batch, uint32_t height, + uint32_t width, uint32_t channels, + uint32_t input_stride, uint32_t output_stride) { + uint32_t n; + if (input == NULL || output == NULL || batch == 0u || height == 0u || + width == 0u || channels == 0u || input_stride == 0u || output_stride == 0u) return; + for (n = 0u; n < batch; ++n) { + uint32_t channel = 0u; +#if LW_NHWC_SUPPORT_X86 + for (; channel + 8u <= channels; channel += 8u) { + __m256 sum = _mm256_setzero_ps(); + for (uint32_t y = 0u; y < height; ++y) { + for (uint32_t x = 0u; x < width; ++x) { + const float* source = input + ((((size_t)n * height + y) * width + x) * input_stride + channel); + sum = _mm256_add_ps(sum, _mm256_loadu_ps(source)); + } + } + sum = _mm256_div_ps(sum, _mm256_set1_ps((float)height * (float)width)); + _mm256_storeu_ps(output + (size_t)n * output_stride + channel, sum); + } +#endif + for (; channel < channels; ++channel) { + float sum = 0.0f; + for (uint32_t y = 0u; y < height; ++y) { + for (uint32_t x = 0u; x < width; ++x) { + sum += input[(((size_t)n * height + y) * width + x) * input_stride + channel]; + } + } + output[(size_t)n * output_stride + channel] = sum / ((float)height * (float)width); + } + } +} + +LW_NHWC_SUPPORT_TARGET +void lw_avx2_nhwc_pool_f32(const float* input, float* output, + uint32_t batch, uint32_t input_height, + uint32_t input_width, uint32_t output_height, + uint32_t output_width, uint32_t channels, + uint32_t kernel_h, uint32_t kernel_w, + uint32_t stride_h, uint32_t stride_w, + uint32_t pad_top, uint32_t pad_left, + uint8_t count_include_pad, uint8_t is_max) { + if (input == NULL || output == NULL || batch == 0u || input_height == 0u || + input_width == 0u || output_height == 0u || output_width == 0u || + channels == 0u || kernel_h == 0u || kernel_w == 0u || + stride_h == 0u || stride_w == 0u) return; + for (uint32_t n = 0u; n < batch; ++n) { + for (uint32_t oy = 0u; oy < output_height; ++oy) { + for (uint32_t ox = 0u; ox < output_width; ++ox) { + int32_t iy0 = (int32_t)((uint64_t)oy * stride_h) - (int32_t)pad_top; + int32_t ix0 = (int32_t)((uint64_t)ox * stride_w) - (int32_t)pad_left; + uint32_t ky_begin; + uint32_t ky_end; + uint32_t kx_begin; + uint32_t kx_end; + uint32_t valid_count; + uint32_t denominator; + uint32_t channel = 0u; + lw_nhwc_pool_valid_window(iy0, ix0, input_height, input_width, + kernel_h, kernel_w, &ky_begin, &ky_end, + &kx_begin, &kx_end); + valid_count = (ky_end - ky_begin) * (kx_end - kx_begin); + denominator = count_include_pad ? kernel_h * kernel_w : valid_count; + if (denominator == 0u) denominator = 1u; +#if LW_NHWC_SUPPORT_X86 + for (; channel + 8u <= channels; channel += 8u) { + __m256 value = is_max ? _mm256_set1_ps(-FLT_MAX) : _mm256_setzero_ps(); + for (uint32_t ky = ky_begin; ky < ky_end; ++ky) { + uint32_t iy = (uint32_t)(iy0 + (int32_t)ky); + for (uint32_t kx = kx_begin; kx < kx_end; ++kx) { + uint32_t ix = (uint32_t)(ix0 + (int32_t)kx); + const float* source = input + ((((size_t)n * input_height + iy) * + input_width + ix) * channels + channel); + __m256 sample = _mm256_loadu_ps(source); + value = is_max ? _mm256_max_ps(value, sample) : _mm256_add_ps(value, sample); + } + } + if (!is_max) { + value = _mm256_mul_ps(value, _mm256_set1_ps(1.0f / (float)denominator)); + } + _mm256_storeu_ps(output + ((((size_t)n * output_height + oy) * output_width + ox) * + channels + channel), value); + } +#endif + for (; channel < channels; ++channel) { + float value = is_max ? -FLT_MAX : 0.0f; + for (uint32_t ky = ky_begin; ky < ky_end; ++ky) { + uint32_t iy = (uint32_t)(iy0 + (int32_t)ky); + for (uint32_t kx = kx_begin; kx < kx_end; ++kx) { + uint32_t ix = (uint32_t)(ix0 + (int32_t)kx); + float sample = input[(((size_t)n * input_height + iy) * + input_width + ix) * channels + channel]; + if (is_max) { + if (sample > value) value = sample; + } else { + value += sample; + } + } + } + if (!is_max) value /= (float)denominator; + output[((((size_t)n * output_height + oy) * output_width + ox) * + channels + channel)] = value; + } + } + } + } +} + +#if LW_NHWC_SUPPORT_X86 +LW_NHWC_SUPPORT_TARGET +static void resize_copy_row(const float* source, float* destination, uint32_t input_width, + uint32_t channels, uint32_t scale_w) { + uint32_t input_x; + for (input_x = 0u; input_x < input_width; ++input_x) { + const float* pixel = source + (size_t)input_x * channels; + float* output_pixel = destination + (size_t)input_x * scale_w * channels; + uint32_t repeat; + for (repeat = 0u; repeat < scale_w; ++repeat) { + uint32_t channel = 0u; + while (channel + 8u <= channels) { + _mm256_storeu_ps(output_pixel + channel, + _mm256_loadu_ps(pixel + channel)); + channel += 8u; + } + while (channel < channels) { + output_pixel[channel] = pixel[channel]; + ++channel; + } + output_pixel += channels; + } + } +} +#endif + +void lw_avx2_nhwc_resize_nearest_f32(const float* input, float* output, + uint32_t batch, uint32_t channels, + uint32_t input_height, uint32_t input_width, + uint32_t output_height, uint32_t output_width) { + uint32_t scale_h; + uint32_t scale_w; + uint32_t n; + uint32_t output_y; + if (input == NULL || output == NULL || input == output || batch == 0u || + channels == 0u || input_height == 0u || input_width == 0u || + output_height < input_height || output_width < input_width || + output_height % input_height != 0u || output_width % input_width != 0u) { + return; + } + scale_h = output_height / input_height; + scale_w = output_width / input_width; + for (n = 0u; n < batch; ++n) { + const float* input_batch = input + (size_t)n * input_height * input_width * channels; + float* output_batch = output + (size_t)n * output_height * output_width * channels; + for (output_y = 0u; output_y < output_height; ++output_y) { + uint32_t source_y = output_y / scale_h; + const float* source_row = + input_batch + (size_t)source_y * input_width * channels; + float* output_row = + output_batch + (size_t)output_y * output_width * channels; +#if LW_NHWC_SUPPORT_X86 + resize_copy_row(source_row, output_row, input_width, channels, scale_w); +#else + { + uint32_t input_x; + for (input_x = 0u; input_x < input_width; ++input_x) { + uint32_t repeat; + for (repeat = 0u; repeat < scale_w; ++repeat) { + memcpy(output_row + ((size_t)input_x * scale_w + repeat) * channels, + source_row + (size_t)input_x * channels, + (size_t)channels * sizeof(float)); + } + } + } +#endif + } + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_packed_conv1x1.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_packed_conv1x1.c new file mode 100644 index 00000000..5c261db3 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_packed_conv1x1.c @@ -0,0 +1,213 @@ +#include "simd_kernels.h" + +#include "packed_conv_internal.h" + +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_PACKED_CONV1X1 1 +#else +# define LW_COMPILES_AVX2_PACKED_CONV1X1 0 +#endif + +#if LW_COMPILES_AVX2_PACKED_CONV1X1 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_packed_conv1x1_f32(const float* input, const float* packed_weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_AVX2_PACKED_CONV1X1 + uint32_t input_channels = (uint32_t)input_dimensions[1]; + uint32_t output_channels = (uint32_t)output_dimensions[1]; + uint64_t channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + uint32_t output_blocks = + (output_channels + LW_PACKED_CONV1X1_OUTPUT_TILE - 1u) / LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_block; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + uint32_t output_base = output_block * LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t valid_outputs = output_channels - output_base; + uint64_t spatial = 0u; + if (valid_outputs > LW_PACKED_CONV1X1_OUTPUT_TILE) { + valid_outputs = LW_PACKED_CONV1X1_OUTPUT_TILE; + } +# if defined(_M_X64) || defined(__x86_64__) + /* x64 exposes sixteen YMM registers, so two spatial vectors can + * stay live for each of the four output planes. This halves the + * packed-weight broadcasts and loop bookkeeping compared with the + * 4x8 kernel below. x86 retains 4x8 because it has only eight + * architectural vector registers and would otherwise spill. */ + if (valid_outputs == LW_PACKED_CONV1X1_OUTPUT_TILE) { + for (; spatial + 16u <= channel_plane; spatial += 16u) { + __m256 accumulator0_low = + _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base]); + __m256 accumulator0_high = accumulator0_low; + __m256 accumulator1_low = + _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 1u]); + __m256 accumulator1_high = accumulator1_low; + __m256 accumulator2_low = + _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 2u]); + __m256 accumulator2_high = accumulator2_low; + __m256 accumulator3_low = + _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base + 3u]); + __m256 accumulator3_high = accumulator3_low; + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * channel_plane + spatial); + const float* packed = + packed_weights + + (size_t)(((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE); + __m256 input_low = _mm256_loadu_ps(input_channel_data); + __m256 input_high = _mm256_loadu_ps(input_channel_data + 8u); + __m256 weight = _mm256_set1_ps(packed[0]); + accumulator0_low = + _mm256_add_ps(accumulator0_low, _mm256_mul_ps(input_low, weight)); + accumulator0_high = + _mm256_add_ps(accumulator0_high, _mm256_mul_ps(input_high, weight)); + weight = _mm256_set1_ps(packed[1]); + accumulator1_low = + _mm256_add_ps(accumulator1_low, _mm256_mul_ps(input_low, weight)); + accumulator1_high = + _mm256_add_ps(accumulator1_high, _mm256_mul_ps(input_high, weight)); + weight = _mm256_set1_ps(packed[2]); + accumulator2_low = + _mm256_add_ps(accumulator2_low, _mm256_mul_ps(input_low, weight)); + accumulator2_high = + _mm256_add_ps(accumulator2_high, _mm256_mul_ps(input_high, weight)); + weight = _mm256_set1_ps(packed[3]); + accumulator3_low = + _mm256_add_ps(accumulator3_low, _mm256_mul_ps(input_low, weight)); + accumulator3_high = + _mm256_add_ps(accumulator3_high, _mm256_mul_ps(input_high, weight)); + } + _mm256_storeu_ps(batch_output + + (size_t)((uint64_t)output_base * channel_plane + spatial), + accumulator0_low); + _mm256_storeu_ps(batch_output + (size_t)((uint64_t)output_base * channel_plane + + spatial + 8u), + accumulator0_high); + _mm256_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 1u) * channel_plane + spatial), + accumulator1_low); + _mm256_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 1u) * channel_plane + spatial + 8u), + accumulator1_high); + _mm256_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 2u) * channel_plane + spatial), + accumulator2_low); + _mm256_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 2u) * channel_plane + spatial + 8u), + accumulator2_high); + _mm256_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 3u) * channel_plane + spatial), + accumulator3_low); + _mm256_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 3u) * channel_plane + spatial + 8u), + accumulator3_high); + } + } +# endif + for (; spatial + 8u <= channel_plane; spatial += 8u) { + __m256 accumulator0 = _mm256_set1_ps(bias == NULL ? 0.0f : bias[output_base]); + __m256 accumulator1 = _mm256_set1_ps( + bias == NULL || valid_outputs <= 1u ? 0.0f : bias[output_base + 1u]); + __m256 accumulator2 = _mm256_set1_ps( + bias == NULL || valid_outputs <= 2u ? 0.0f : bias[output_base + 2u]); + __m256 accumulator3 = _mm256_set1_ps( + bias == NULL || valid_outputs <= 3u ? 0.0f : bias[output_base + 3u]); + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane + spatial); + const float* packed = + packed_weights + + (size_t)(((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE); + __m256 input_values = _mm256_loadu_ps(input_channel_data); + accumulator0 = _mm256_add_ps( + accumulator0, _mm256_mul_ps(input_values, _mm256_set1_ps(packed[0]))); + accumulator1 = _mm256_add_ps( + accumulator1, _mm256_mul_ps(input_values, _mm256_set1_ps(packed[1]))); + accumulator2 = _mm256_add_ps( + accumulator2, _mm256_mul_ps(input_values, _mm256_set1_ps(packed[2]))); + accumulator3 = _mm256_add_ps( + accumulator3, _mm256_mul_ps(input_values, _mm256_set1_ps(packed[3]))); + } + _mm256_storeu_ps(batch_output + + (size_t)((uint64_t)output_base * channel_plane + spatial), + accumulator0); + if (valid_outputs > 1u) { + _mm256_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 1u) * channel_plane + spatial), + accumulator1); + } + if (valid_outputs > 2u) { + _mm256_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 2u) * channel_plane + spatial), + accumulator2); + } + if (valid_outputs > 3u) { + _mm256_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 3u) * channel_plane + spatial), + accumulator3); + } + } + for (; spatial < channel_plane; ++spatial) { + float accumulator0 = bias == NULL ? 0.0f : bias[output_base]; + float accumulator1 = + bias == NULL || valid_outputs <= 1u ? 0.0f : bias[output_base + 1u]; + float accumulator2 = + bias == NULL || valid_outputs <= 2u ? 0.0f : bias[output_base + 2u]; + float accumulator3 = + bias == NULL || valid_outputs <= 3u ? 0.0f : bias[output_base + 3u]; + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + float input_value = + batch_input[(size_t)((uint64_t)input_channel * channel_plane + spatial)]; + const float* packed = + packed_weights + + (size_t)(((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE); + accumulator0 += input_value * packed[0]; + accumulator1 += input_value * packed[1]; + accumulator2 += input_value * packed[2]; + accumulator3 += input_value * packed[3]; + } + batch_output[(size_t)((uint64_t)output_base * channel_plane + spatial)] = + accumulator0; + if (valid_outputs > 1u) + batch_output[(size_t)((uint64_t)(output_base + 1u) * channel_plane + spatial)] = + accumulator1; + if (valid_outputs > 2u) + batch_output[(size_t)((uint64_t)(output_base + 2u) * channel_plane + spatial)] = + accumulator2; + if (valid_outputs > 3u) + batch_output[(size_t)((uint64_t)(output_base + 3u) * channel_plane + spatial)] = + accumulator3; + } + } + } +#else + lw_scalar_packed_conv1x1_f32(input, packed_weights, bias, output, input_dimensions, + output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_softmax.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_softmax.c new file mode 100644 index 00000000..065aef76 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_softmax.c @@ -0,0 +1,351 @@ +#include "simd_kernels.h" + +/* AVX2 Softmax for the long, contiguous REC classification axis. */ + +#include "../kernels/packed_matmul_internal.h" + +#include +#include +#include + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +# include +# define LW_COMPILES_AVX2_SOFTMAX 1 +#else +# define LW_COMPILES_AVX2_SOFTMAX 0 +#endif + +#if LW_COMPILES_AVX2_SOFTMAX +# if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2,no-fma"))) +# endif +static __m256 +exp_approximation_f32(__m256 value) { + /* This is the standard base-2 range reduction used by vector math + * kernels. The clamp matches the finite range accepted by expf and the + * polynomial error stays well below the public graph tolerances. */ + const __m256 log2e = _mm256_set1_ps(1.44269504088896341f); + const __m256 c1 = _mm256_set1_ps(0.693359375f); + const __m256 c2 = _mm256_set1_ps(-2.12194440e-4f); + const __m256 minimum = _mm256_set1_ps(-88.3762626647949f); + const __m256 maximum = _mm256_set1_ps(88.3762626647949f); + const __m256 half = _mm256_set1_ps(0.5f); + __m256 scaled; + __m256 integer_part; + __m256 reduced; + __m256 square; + __m256 polynomial; + __m256i exponent; + __m256 power; + + value = _mm256_max_ps(value, minimum); + value = _mm256_min_ps(value, maximum); + scaled = _mm256_add_ps(_mm256_mul_ps(value, log2e), half); + integer_part = _mm256_cvtepi32_ps(_mm256_cvttps_epi32(scaled)); + /* Truncation toward zero rounds negative values in the wrong direction. */ + integer_part = + _mm256_sub_ps(integer_part, _mm256_and_ps(_mm256_cmp_ps(integer_part, scaled, _CMP_GT_OQ), + _mm256_set1_ps(1.0f))); + reduced = _mm256_sub_ps(value, _mm256_mul_ps(integer_part, c1)); + reduced = _mm256_sub_ps(reduced, _mm256_mul_ps(integer_part, c2)); + square = _mm256_mul_ps(reduced, reduced); + /* Taylor expansion around zero. Range reduction keeps |reduced| below + * ln(2)/2, so the omitted ninth-order term is below 1e-8 in FP32. */ + polynomial = _mm256_set1_ps(2.4801587302E-5f); /* 1 / 40320 */ + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.9841269841E-4f), _mm256_mul_ps(reduced, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.3888888889E-3f), _mm256_mul_ps(reduced, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(8.3333333333E-3f), _mm256_mul_ps(reduced, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(4.1666666667E-2f), _mm256_mul_ps(reduced, polynomial)); + polynomial = + _mm256_add_ps(_mm256_set1_ps(1.6666666667E-1f), _mm256_mul_ps(reduced, polynomial)); + polynomial = _mm256_add_ps(_mm256_set1_ps(5.0E-1f), _mm256_mul_ps(reduced, polynomial)); + polynomial = _mm256_add_ps(reduced, _mm256_mul_ps(square, polynomial)); + polynomial = _mm256_add_ps(_mm256_set1_ps(1.0f), polynomial); + exponent = _mm256_add_epi32(_mm256_cvttps_epi32(integer_part), _mm256_set1_epi32(127)); + power = _mm256_castsi256_ps(_mm256_slli_epi32(exponent, 23)); + return _mm256_mul_ps(polynomial, power); +} +#endif + +#if LW_COMPILES_AVX2_SOFTMAX && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_softmax_contiguous_f32(const float* input, float* output, uint64_t row_count, + uint64_t axis_count) { +#if LW_COMPILES_AVX2_SOFTMAX + uint64_t row; + for (row = 0u; row < row_count; ++row) { + const float* input_row = input + (size_t)(row * axis_count); + float* output_row = output + (size_t)(row * axis_count); + float maximum = input_row[0]; + float sum; + uint64_t index; + __m256 maximum_values = _mm256_set1_ps(input_row[0]); + for (index = 0u; index + 8u <= axis_count; index += 8u) { + maximum_values = + _mm256_max_ps(maximum_values, _mm256_loadu_ps(input_row + (size_t)index)); + } + for (; index < axis_count; ++index) { + if (input_row[(size_t)index] > maximum) { + maximum = input_row[(size_t)index]; + } + } + { + float lanes[8]; + _mm256_storeu_ps(lanes, maximum_values); + for (index = 0u; index < 8u; ++index) { + if (lanes[index] > maximum) { + maximum = lanes[index]; + } + } + } + sum = 0.0f; + index = 0u; + { + __m256 maximum_vector = _mm256_set1_ps(maximum); + for (; index + 8u <= axis_count; index += 8u) { + __m256 values = exp_approximation_f32( + _mm256_sub_ps(_mm256_loadu_ps(input_row + (size_t)index), maximum_vector)); + _mm256_storeu_ps(output_row + (size_t)index, values); + } + } + for (index = axis_count - (axis_count % 8u); index < axis_count; ++index) { + float value = expf(input_row[(size_t)index] - maximum); + output_row[(size_t)index] = value; + } + /* Keep the established left-to-right accumulation order so callers + * that consume the maximum probability retain their score contract. */ + for (index = 0u; index < axis_count; ++index) { + sum += output_row[(size_t)index]; + } + { + __m256 inverse_sum = _mm256_set1_ps(1.0f / sum); + for (index = 0u; index + 8u <= axis_count; index += 8u) { + _mm256_storeu_ps( + output_row + (size_t)index, + _mm256_mul_ps(_mm256_loadu_ps(output_row + (size_t)index), inverse_sum)); + } + } + for (index = axis_count - (axis_count % 8u); index < axis_count; ++index) { + output_row[(size_t)index] /= sum; + } + } +#else + uint64_t row; + for (row = 0u; row < row_count; ++row) { + const float* input_row = input + (size_t)(row * axis_count); + float* output_row = output + (size_t)(row * axis_count); + float maximum = input_row[0]; + float sum = 0.0f; + uint64_t index; + for (index = 1u; index < axis_count; ++index) { + if (input_row[(size_t)index] > maximum) { + maximum = input_row[(size_t)index]; + } + } + for (index = 0u; index < axis_count; ++index) { + float value = expf(input_row[(size_t)index] - maximum); + output_row[(size_t)index] = value; + sum += value; + } + for (index = 0u; index < axis_count; ++index) { + output_row[(size_t)index] /= sum; + } + } +#endif +} + +#if LW_COMPILES_AVX2_SOFTMAX && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,no-fma"))) +#endif +void lw_avx2_ctc_emitted_softmax_contiguous_f32(const float* input, + const uint32_t* best_indices, + float* emitted_probabilities, + uint64_t row_count, uint64_t axis_count) { +#if LW_COMPILES_AVX2_SOFTMAX + uint64_t row; + for (row = 0u; row < row_count; ++row) { + uint32_t best_index = best_indices[(size_t)row]; + if (best_index != 0u && + (row == 0u || best_index != best_indices[(size_t)row - 1u])) { + const float* input_row = input + (size_t)(row * axis_count); + float maximum = input_row[best_index]; + float sum = 0.0f; + uint64_t index; + __m256 maximum_vector = _mm256_set1_ps(maximum); + for (index = 0u; index + 8u <= axis_count; index += 8u) { + float lanes[8]; + __m256 values = exp_approximation_f32( + _mm256_sub_ps(_mm256_loadu_ps(input_row + (size_t)index), maximum_vector)); + uint32_t lane; + _mm256_storeu_ps(lanes, values); + for (lane = 0u; lane < 8u; ++lane) { + sum += lanes[lane]; + } + } + for (; index < axis_count; ++index) { + sum += expf(input_row[(size_t)index] - maximum); + } + emitted_probabilities[(size_t)row] = 1.0f / sum; + } + } +#else + uint64_t row; + for (row = 0u; row < row_count; ++row) { + uint32_t best_index = best_indices[(size_t)row]; + if (best_index != 0u && + (row == 0u || best_index != best_indices[(size_t)row - 1u])) { + const float* input_row = input + (size_t)(row * axis_count); + float maximum = input_row[best_index]; + float sum = 0.0f; + uint64_t index; + for (index = 0u; index < axis_count; ++index) { + sum += expf(input_row[(size_t)index] - maximum); + } + emitted_probabilities[(size_t)row] = 1.0f / sum; + } + } +#endif +} + +#if LW_COMPILES_AVX2_SOFTMAX && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2,fma"))) +#endif +void lw_avx2_ctc_row_probabilities_f32( + const float* input, const float* packed_weights, const float* bias, + const uint32_t* best_indices, const float* scores, float* probabilities, + uint32_t rows, uint32_t inner_dimension, uint32_t columns) { +#if LW_COMPILES_AVX2_SOFTMAX + const uint32_t column_panels = + (columns + LW_PACKED_MATMUL_COLUMN_TILE - 1u) / LW_PACKED_MATMUL_COLUMN_TILE; + uint32_t row; + /* Only emitted rows need a probability: recompute that row's logits as a + * single packed matvec and fold the softmax denominator into the same + * pass, so the [rows x columns] logit tensor never exists. The 8-lane + * batches and per-lane accumulation order match + * lw_avx2_ctc_emitted_softmax_contiguous_f32 for bit-comparable sums. */ + for (row = 0u; row < rows; ++row) { + uint32_t best_index = best_indices[(size_t)row]; + if (best_index != 0u && + (row == 0u || best_index != best_indices[(size_t)row - 1u])) { + const float maximum = scores[(size_t)row]; + const __m256 maximum_vector = _mm256_set1_ps(maximum); + const float* input_row = input + (size_t)row * inner_dimension; + float sum = 0.0f; + uint32_t column_panel; + for (column_panel = 0u; column_panel < column_panels; ++column_panel) { + const float* packed = + packed_weights + + (size_t)(column_panel * inner_dimension * + LW_PACKED_MATMUL_COLUMN_TILE); + __m256 accumulator_low = _mm256_setzero_ps(); + __m256 accumulator_high = _mm256_setzero_ps(); + uint32_t inner; + for (inner = 0u; inner < inner_dimension; ++inner) { + const __m256 weight_low = _mm256_loadu_ps(packed); + const __m256 weight_high = _mm256_loadu_ps(packed + 8u); + const __m256 input_value = _mm256_set1_ps(input_row[inner]); + accumulator_low = + _mm256_fmadd_ps(input_value, weight_low, accumulator_low); + accumulator_high = + _mm256_fmadd_ps(input_value, weight_high, accumulator_high); + packed += LW_PACKED_MATMUL_COLUMN_TILE; + } + { + const uint32_t column_base = + column_panel * LW_PACKED_MATMUL_COLUMN_TILE; + const uint32_t valid_columns = + columns - column_base < LW_PACKED_MATMUL_COLUMN_TILE + ? columns - column_base + : LW_PACKED_MATMUL_COLUMN_TILE; + if (bias != NULL) { + __m256 bias_low; + __m256 bias_high; + if (valid_columns == LW_PACKED_MATMUL_COLUMN_TILE) { + bias_low = _mm256_loadu_ps(bias + column_base); + bias_high = _mm256_loadu_ps(bias + column_base + 8u); + } else { + /* Packed weights are padded to 16 lanes, but + * model bias tensors contain only real classes. */ + float bias_values[LW_PACKED_MATMUL_COLUMN_TILE] = {0.0f}; + uint32_t lane; + for (lane = 0u; lane < valid_columns; ++lane) { + bias_values[lane] = bias[column_base + lane]; + } + bias_low = _mm256_loadu_ps(bias_values); + bias_high = _mm256_loadu_ps(bias_values + 8u); + } + accumulator_low = _mm256_add_ps( + accumulator_low, bias_low); + accumulator_high = _mm256_add_ps( + accumulator_high, bias_high); + } + if (valid_columns == LW_PACKED_MATMUL_COLUMN_TILE) { + float lanes[8]; + __m256 values; + uint32_t lane; + values = exp_approximation_f32( + _mm256_sub_ps(accumulator_low, maximum_vector)); + _mm256_storeu_ps(lanes, values); + for (lane = 0u; lane < 8u; ++lane) sum += lanes[lane]; + values = exp_approximation_f32( + _mm256_sub_ps(accumulator_high, maximum_vector)); + _mm256_storeu_ps(lanes, values); + for (lane = 0u; lane < 8u; ++lane) sum += lanes[lane]; + } else { + float lanes[LW_PACKED_MATMUL_COLUMN_TILE]; + uint32_t lane = 0u; + _mm256_storeu_ps(lanes, accumulator_low); + _mm256_storeu_ps(lanes + 8u, accumulator_high); + while (lane + 8u <= valid_columns) { + float batch[8]; + __m256 values = exp_approximation_f32(_mm256_sub_ps( + _mm256_loadu_ps(lanes + lane), maximum_vector)); + uint32_t sub_lane; + _mm256_storeu_ps(batch, values); + for (sub_lane = 0u; sub_lane < 8u; ++sub_lane) { + sum += batch[sub_lane]; + } + lane += 8u; + } + for (; lane < valid_columns; ++lane) { + sum += expf(lanes[lane] - maximum); + } + } + } + } + probabilities[(size_t)row] = 1.0f / sum; + } + } +#else + uint32_t row; + for (row = 0u; row < rows; ++row) { + uint32_t best_index = best_indices[(size_t)row]; + if (best_index != 0u && + (row == 0u || best_index != best_indices[(size_t)row - 1u])) { + const float maximum = scores[(size_t)row]; + const float* input_row = input + (size_t)row * inner_dimension; + float sum = 0.0f; + uint32_t column; + for (column = 0u; column < columns; ++column) { + float value = bias != NULL ? bias[column] : 0.0f; + uint32_t inner; + const float* weights = packed_weights + + (size_t)((column / LW_PACKED_MATMUL_COLUMN_TILE) * + inner_dimension * LW_PACKED_MATMUL_COLUMN_TILE + + (column % LW_PACKED_MATMUL_COLUMN_TILE)); + for (inner = 0u; inner < inner_dimension; ++inner) { + value += input_row[inner] * + weights[(size_t)inner * LW_PACKED_MATMUL_COLUMN_TILE]; + } + sum += expf(value - maximum); + } + probabilities[(size_t)row] = 1.0f / sum; + } + } +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_transpose.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_transpose.c new file mode 100644 index 00000000..45e88d93 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/avx2_transpose.c @@ -0,0 +1,100 @@ +#include "simd_kernels.h" + +#include + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) +#define LW_TRANSPOSE_X86 1 +#include +#else +#define LW_TRANSPOSE_X86 0 +#endif + +#if LW_TRANSPOSE_X86 && (defined(__GNUC__) || defined(__clang__)) +#define LW_TRANSPOSE_AVX2 __attribute__((target("avx2"))) +#else +#define LW_TRANSPOSE_AVX2 +#endif + +#if LW_TRANSPOSE_X86 +/* In-register 8x8 float transpose. Pure data movement: bit-identical to any + * scalar reordering of the same elements. */ +LW_TRANSPOSE_AVX2 +static void transpose8x8_f32(__m256 rows[8]) { + __m256 t0 = _mm256_unpacklo_ps(rows[0], rows[1]); + __m256 t1 = _mm256_unpackhi_ps(rows[0], rows[1]); + __m256 t2 = _mm256_unpacklo_ps(rows[2], rows[3]); + __m256 t3 = _mm256_unpackhi_ps(rows[2], rows[3]); + __m256 t4 = _mm256_unpacklo_ps(rows[4], rows[5]); + __m256 t5 = _mm256_unpackhi_ps(rows[4], rows[5]); + __m256 t6 = _mm256_unpacklo_ps(rows[6], rows[7]); + __m256 t7 = _mm256_unpackhi_ps(rows[6], rows[7]); + __m256 u0 = _mm256_shuffle_ps(t0, t2, 0x44); + __m256 u1 = _mm256_shuffle_ps(t0, t2, 0xEE); + __m256 u2 = _mm256_shuffle_ps(t1, t3, 0x44); + __m256 u3 = _mm256_shuffle_ps(t1, t3, 0xEE); + __m256 u4 = _mm256_shuffle_ps(t4, t6, 0x44); + __m256 u5 = _mm256_shuffle_ps(t4, t6, 0xEE); + __m256 u6 = _mm256_shuffle_ps(t5, t7, 0x44); + __m256 u7 = _mm256_shuffle_ps(t5, t7, 0xEE); + rows[0] = _mm256_permute2f128_ps(u0, u4, 0x20); + rows[1] = _mm256_permute2f128_ps(u1, u5, 0x20); + rows[2] = _mm256_permute2f128_ps(u2, u6, 0x20); + rows[3] = _mm256_permute2f128_ps(u3, u7, 0x20); + rows[4] = _mm256_permute2f128_ps(u0, u4, 0x31); + rows[5] = _mm256_permute2f128_ps(u1, u5, 0x31); + rows[6] = _mm256_permute2f128_ps(u2, u6, 0x31); + rows[7] = _mm256_permute2f128_ps(u3, u7, 0x31); +} +#endif + +/* Transpose a rows x cols row-major matrix into a cols x rows row-major + * matrix: output[col * rows + row] = input[row * cols + col]. */ +LW_TRANSPOSE_AVX2 +void lw_avx2_transpose_2d_f32(const float* input, float* output, + uint32_t rows, uint32_t cols) { +#if LW_TRANSPOSE_X86 + uint32_t row = 0u; + uint32_t col; + uint32_t r; + uint32_t c; + if (input == NULL || output == NULL) { + return; + } + for (; row + 8u <= rows; row += 8u) { + for (col = 0u; col + 8u <= cols; col += 8u) { + __m256 block[8]; + uint32_t k; + for (k = 0u; k < 8u; ++k) { + block[k] = _mm256_loadu_ps(input + (size_t)(row + k) * cols + col); + } + transpose8x8_f32(block); + for (k = 0u; k < 8u; ++k) { + _mm256_storeu_ps(output + (size_t)(col + k) * rows + row, block[k]); + } + } + /* Right edge: remaining columns for the blocked rows. */ + for (; col < cols; ++col) { + for (r = 0u; r < 8u; ++r) { + output[(size_t)col * rows + row + r] = input[(size_t)(row + r) * cols + col]; + } + } + } + /* Bottom edge: remaining rows across all columns. */ + for (; row < rows; ++row) { + for (c = 0u; c < cols; ++c) { + output[(size_t)c * rows + row] = input[(size_t)row * cols + c]; + } + } +#else + uint32_t r; + uint32_t c; + if (input == NULL || output == NULL) { + return; + } + for (r = 0u; r < rows; ++r) { + for (c = 0u; c < cols; ++c) { + output[(size_t)c * rows + r] = input[(size_t)r * cols + c]; + } + } +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/cpu_features.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/cpu_features.c new file mode 100644 index 00000000..b0dbbf49 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/cpu_features.c @@ -0,0 +1,231 @@ +#include "cpu_features.h" + +/* + * One-time runtime CPU detection. Compiling AVX2 objects does not mean the host + * CPU supports AVX2, so callers must always dispatch through this result. + */ + +#include + +#if defined(__EMSCRIPTEN__) && defined(__wasm_simd128__) +# define LW_EMSCRIPTEN_SIMD128 1 +#else +# define LW_EMSCRIPTEN_SIMD128 0 +#endif + +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) +# include +#elif defined(__i386__) || defined(__x86_64__) +# include +#endif + +#if defined(__linux__) && defined(__loongarch__) +# include +# include +# ifndef HWCAP_LOONGARCH_LSX +# define HWCAP_LOONGARCH_LSX (1UL << 4) +# endif +# ifndef HWCAP_LOONGARCH_LASX +# define HWCAP_LOONGARCH_LASX (1UL << 5) +# endif +#endif + +static lw_simd_level lw_detect_simd_level_uncached(void) { +#if LW_EMSCRIPTEN_SIMD128 + /* The module itself requires SIMD128, so no runtime CPUID probe exists. */ + return LW_SIMD_LEVEL_SSE2; +#elif defined(_M_ARM64) || defined(__aarch64__) + /* Advanced SIMD is part of the AArch64 execution environment. */ + return LW_SIMD_LEVEL_NEON; +#elif defined(__linux__) && defined(__loongarch__) + { + const unsigned long capabilities = getauxval(AT_HWCAP); + if ((capabilities & HWCAP_LOONGARCH_LASX) != 0u) { + return LW_SIMD_LEVEL_LASX; + } + if ((capabilities & HWCAP_LOONGARCH_LSX) != 0u) { + return LW_SIMD_LEVEL_LSX; + } + return LW_SIMD_LEVEL_SCALAR; + } +#elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) + int registers[4]; + int maximum_leaf; + int has_sse2; + __cpuid(registers, 0); + maximum_leaf = registers[0]; + if (maximum_leaf < 1) { + return LW_SIMD_LEVEL_SCALAR; + } + __cpuid(registers, 1); + has_sse2 = (registers[3] & (1 << 26)) != 0; + /* AVX2 is safe only when both hardware and the operating system preserve + * XMM/YMM state. XGETBV verifies the OS-enabled extended-state bits. */ + if (maximum_leaf >= 7 && (registers[2] & (1 << 27)) != 0 && (registers[2] & (1 << 28)) != 0 && + (_xgetbv(0) & 6u) == 6u) { + __cpuidex(registers, 7, 0); + if ((registers[1] & (1 << 5)) != 0) { + return LW_SIMD_LEVEL_AVX2; + } + } + return has_sse2 ? LW_SIMD_LEVEL_SSE2 : LW_SIMD_LEVEL_SCALAR; +#elif defined(__i386__) || defined(__x86_64__) + unsigned int eax; + unsigned int ebx; + unsigned int ecx; + unsigned int edx; + unsigned int maximum_leaf = __get_cpuid_max(0u, NULL); + int has_sse2; + if (maximum_leaf < 1u || __get_cpuid(1u, &eax, &ebx, &ecx, &edx) == 0) { + return LW_SIMD_LEVEL_SCALAR; + } + has_sse2 = (edx & bit_SSE2) != 0u; + /* GCC/Clang branch performs the same hardware + OS state check. */ + if (maximum_leaf >= 7u && (ecx & bit_OSXSAVE) != 0u && (ecx & bit_AVX) != 0u) { + unsigned int xcr0_eax; + unsigned int xcr0_edx; + __asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0_eax), "=d"(xcr0_edx) : "c"(0u)); + (void)xcr0_edx; + if ((xcr0_eax & 6u) == 6u) { + __cpuid_count(7u, 0u, eax, ebx, ecx, edx); + if ((ebx & bit_AVX2) != 0u) { + return LW_SIMD_LEVEL_AVX2; + } + } + } + return has_sse2 ? LW_SIMD_LEVEL_SSE2 : LW_SIMD_LEVEL_SCALAR; +#else + return LW_SIMD_LEVEL_SCALAR; +#endif +} + +lw_simd_level lw_detect_simd_level(void) { +#if defined(_MSC_VER) + static volatile long cached = -1; + long value = _InterlockedCompareExchange(&cached, -1, -1); + if (value >= 0) { + return (lw_simd_level)value; + } + value = (long)lw_detect_simd_level_uncached(); + if (_InterlockedCompareExchange(&cached, value, -1) != -1) { + value = _InterlockedCompareExchange(&cached, -1, -1); + } + return (lw_simd_level)value; +#elif defined(__GNUC__) || defined(__clang__) + static int cached = -1; + int value = __atomic_load_n(&cached, __ATOMIC_ACQUIRE); + int expected = -1; + if (value >= 0) { + return (lw_simd_level)value; + } + value = (int)lw_detect_simd_level_uncached(); + (void)__atomic_compare_exchange_n( + &cached, &expected, value, 0, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE); + return (lw_simd_level)__atomic_load_n(&cached, __ATOMIC_ACQUIRE); +#else + static int cached = -1; + if (cached < 0) { + cached = (int)lw_detect_simd_level_uncached(); + } + return (lw_simd_level)cached; +#endif +} +const char* lw_simd_level_name(lw_simd_level level) { + if (level == LW_SIMD_LEVEL_AVX2) { + return "avx2"; + } + if (level == LW_SIMD_LEVEL_SSE2) { + return "sse2"; + } + if (level == LW_SIMD_LEVEL_NEON) { + return "neon"; + } + if (level == LW_SIMD_LEVEL_LSX) { + return "lsx"; + } + if (level == LW_SIMD_LEVEL_LASX) { + return "lasx"; + } + return "scalar"; +} + +static int lw_detect_fma(lw_simd_level simd) { + (void)simd; +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) + int registers[4]; + int maximum_leaf; + __cpuid(registers, 0); + maximum_leaf = registers[0]; + if (maximum_leaf < 1) { + return 0; + } + __cpuid(registers, 1); + if ((registers[2] & (1 << 12)) == 0 || (registers[2] & (1 << 27)) == 0 || + (registers[2] & (1 << 28)) == 0 || (_xgetbv(0) & 6u) != 6u) { + return 0; + } + return 1; +#elif (defined(__i386__) || defined(__x86_64__)) && !defined(__EMSCRIPTEN__) + unsigned int eax; + unsigned int ebx; + unsigned int ecx; + unsigned int edx; + unsigned int maximum_leaf; + unsigned int xcr0_eax; + unsigned int xcr0_edx; + maximum_leaf = __get_cpuid_max(0u, NULL); + if (maximum_leaf < 1u || __get_cpuid(1u, &eax, &ebx, &ecx, &edx) == 0) { + return 0; + } + if ((ecx & bit_FMA) == 0u || (ecx & bit_OSXSAVE) == 0u || (ecx & bit_AVX) == 0u) { + return 0; + } + __asm__ volatile(".byte 0x0f, 0x01, 0xd0" + : "=a"(xcr0_eax), "=d"(xcr0_edx) + : "c"(0u)); + (void)xcr0_edx; + return (xcr0_eax & 6u) == 6u; +#else + (void)simd; + return 0; +#endif +} + +lw_cpu_capabilities lw_get_cpu_capabilities(void) { + static lw_cpu_capabilities cached; +#if defined(_MSC_VER) + static volatile long state; + long observed = _InterlockedCompareExchange(&state, 1, 0); + if (observed == 0) { + cached.simd = lw_detect_simd_level(); + cached.has_fma = lw_detect_fma(cached.simd); + cached.has_avx2_fma = lw_simd_level_is_avx2(cached.simd) && cached.has_fma; + _InterlockedExchange(&state, 2); + } else { + while (_InterlockedCompareExchange(&state, 2, 2) != 2) { + } + } +#elif defined(__GNUC__) || defined(__clang__) + static int state; + int expected = 0; + if (__atomic_compare_exchange_n( + &state, &expected, 1, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { + cached.simd = lw_detect_simd_level(); + cached.has_fma = lw_detect_fma(cached.simd); + cached.has_avx2_fma = lw_simd_level_is_avx2(cached.simd) && cached.has_fma; + __atomic_store_n(&state, 2, __ATOMIC_RELEASE); + } else { + while (__atomic_load_n(&state, __ATOMIC_ACQUIRE) != 2) { + } + } +#else + static int initialized; + if (!initialized) { + cached.simd = lw_detect_simd_level(); + cached.has_fma = lw_detect_fma(cached.simd); + cached.has_avx2_fma = lw_simd_level_is_avx2(cached.simd) && cached.has_fma; + initialized = 1; + } +#endif + return cached; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/cpu_features.h b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/cpu_features.h new file mode 100644 index 00000000..185f96c7 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/cpu_features.h @@ -0,0 +1,58 @@ +#ifndef LW_CPU_FEATURES_H +#define LW_CPU_FEATURES_H + +/* + * Backends are mutually exclusive architecture implementations, not an + * ordered capability level. Callers must use the helpers below instead of + * comparing enum values: NEON and LSX are not supersets of SSE2/AVX2. + */ + +typedef enum lw_simd_level { + LW_SIMD_LEVEL_SCALAR = 0, + LW_SIMD_LEVEL_SSE2 = 1, + LW_SIMD_LEVEL_AVX2 = 2, + LW_SIMD_LEVEL_NEON = 3, + LW_SIMD_LEVEL_LSX = 4, + LW_SIMD_LEVEL_LASX = 5 +} lw_simd_level; + +static inline int lw_simd_level_is_sse2(lw_simd_level level) { + return level == LW_SIMD_LEVEL_SSE2 || level == LW_SIMD_LEVEL_AVX2; +} + +static inline int lw_simd_level_is_avx2(lw_simd_level level) { + return level == LW_SIMD_LEVEL_AVX2; +} + +static inline int lw_simd_level_is_neon(lw_simd_level level) { + return level == LW_SIMD_LEVEL_NEON; +} + +static inline int lw_simd_level_is_lsx(lw_simd_level level) { + /* Every LASX processor also provides the 128-bit LSX instruction set. */ + return level == LW_SIMD_LEVEL_LSX || level == LW_SIMD_LEVEL_LASX; +} + +static inline int lw_simd_level_has_packed_conv1x1(lw_simd_level level) { + return lw_simd_level_is_sse2(level) || lw_simd_level_is_neon(level) || + lw_simd_level_is_lsx(level); +} + +lw_simd_level lw_detect_simd_level(void); +const char* lw_simd_level_name(lw_simd_level level); + +/* + * SIMD backends are mutually exclusive, while these flags describe optional + * capabilities layered on top of a backend. The snapshot is internal-facing; + * the public C ABI and model format do not expose CPU features. + */ +typedef struct lw_cpu_capabilities { + lw_simd_level simd; + int has_fma; + int has_avx2_fma; +} lw_cpu_capabilities; + +/* Detects the host once during session initialization. */ +lw_cpu_capabilities lw_get_cpu_capabilities(void); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/lsx_packed_conv1x1.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/lsx_packed_conv1x1.c new file mode 100644 index 00000000..ff9bb42d --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/lsx_packed_conv1x1.c @@ -0,0 +1,137 @@ +#include "simd_kernels.h" + +#include "packed_conv_internal.h" + +#include + +#if defined(__loongarch_sx) +# include +# define LW_COMPILES_LSX_PACKED_CONV1X1 1 +#else +# define LW_COMPILES_LSX_PACKED_CONV1X1 0 +#endif + +#if LW_COMPILES_LSX_PACKED_CONV1X1 +static __m128 lw_lsx_broadcast_f32(float value) { + return (__m128)__lsx_vldrepl_w(&value, 0); +} + +static __m128 lw_lsx_load_f32(const float* values) { + return (__m128)__lsx_vld(values, 0); +} + +static void lw_lsx_store_f32(float* output, __m128 values) { + __lsx_vst((__m128i)values, output, 0); +} +#endif + +void lw_lsx_packed_conv1x1_f32(const float* input, const float* packed_weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_LSX_PACKED_CONV1X1 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint64_t channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + const uint32_t output_blocks = + (output_channels + LW_PACKED_CONV1X1_OUTPUT_TILE - 1u) / + LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_block; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + const uint32_t output_base = output_block * LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t valid_outputs = output_channels - output_base; + uint64_t spatial = 0u; + if (valid_outputs > LW_PACKED_CONV1X1_OUTPUT_TILE) { + valid_outputs = LW_PACKED_CONV1X1_OUTPUT_TILE; + } + for (; spatial + 4u <= channel_plane; spatial += 4u) { + __m128 accumulator0 = + lw_lsx_broadcast_f32(bias == NULL ? 0.0f : bias[output_base]); + __m128 accumulator1 = lw_lsx_broadcast_f32( + bias == NULL || valid_outputs <= 1u ? 0.0f : bias[output_base + 1u]); + __m128 accumulator2 = lw_lsx_broadcast_f32( + bias == NULL || valid_outputs <= 2u ? 0.0f : bias[output_base + 2u]); + __m128 accumulator3 = lw_lsx_broadcast_f32( + bias == NULL || valid_outputs <= 3u ? 0.0f : bias[output_base + 3u]); + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * channel_plane + spatial); + const float* packed = + packed_weights + + (size_t)(((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE); + const __m128 input_values = lw_lsx_load_f32(input_channel_data); + accumulator0 = __lsx_vfadd_s( + accumulator0, + __lsx_vfmul_s(input_values, lw_lsx_broadcast_f32(packed[0]))); + accumulator1 = __lsx_vfadd_s( + accumulator1, + __lsx_vfmul_s(input_values, lw_lsx_broadcast_f32(packed[1]))); + accumulator2 = __lsx_vfadd_s( + accumulator2, + __lsx_vfmul_s(input_values, lw_lsx_broadcast_f32(packed[2]))); + accumulator3 = __lsx_vfadd_s( + accumulator3, + __lsx_vfmul_s(input_values, lw_lsx_broadcast_f32(packed[3]))); + } + lw_lsx_store_f32(batch_output + + (size_t)((uint64_t)output_base * channel_plane + spatial), + accumulator0); + if (valid_outputs > 1u) { + lw_lsx_store_f32( + batch_output + + (size_t)((uint64_t)(output_base + 1u) * channel_plane + spatial), + accumulator1); + } + if (valid_outputs > 2u) { + lw_lsx_store_f32( + batch_output + + (size_t)((uint64_t)(output_base + 2u) * channel_plane + spatial), + accumulator2); + } + if (valid_outputs > 3u) { + lw_lsx_store_f32( + batch_output + + (size_t)((uint64_t)(output_base + 3u) * channel_plane + spatial), + accumulator3); + } + } + for (; spatial < channel_plane; ++spatial) { + float accumulators[LW_PACKED_CONV1X1_OUTPUT_TILE] = { + bias == NULL ? 0.0f : bias[output_base], + bias == NULL || valid_outputs <= 1u ? 0.0f : bias[output_base + 1u], + bias == NULL || valid_outputs <= 2u ? 0.0f : bias[output_base + 2u], + bias == NULL || valid_outputs <= 3u ? 0.0f : bias[output_base + 3u]}; + uint32_t input_channel; + uint32_t output_lane; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float input_value = + batch_input[(size_t)((uint64_t)input_channel * channel_plane + spatial)]; + const float* packed = + packed_weights + + (size_t)(((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE); + for (output_lane = 0u; output_lane < valid_outputs; ++output_lane) { + accumulators[output_lane] += input_value * packed[output_lane]; + } + } + for (output_lane = 0u; output_lane < valid_outputs; ++output_lane) { + batch_output[(size_t)(((uint64_t)output_base + output_lane) * channel_plane + + spatial)] = accumulators[output_lane]; + } + } + } + } +#else + lw_scalar_packed_conv1x1_f32(input, packed_weights, bias, output, input_dimensions, + output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/neon_conv3x3.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/neon_conv3x3.c new file mode 100644 index 00000000..e55ea9d1 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/neon_conv3x3.c @@ -0,0 +1,91 @@ +#include "simd_kernels.h" + +#include "scalar_kernels.h" + +#include + +#if defined(_M_ARM64) || defined(__aarch64__) +# include +# define LW_COMPILES_NEON_CONV3X3 1 +#else +# define LW_COMPILES_NEON_CONV3X3 0 +#endif + +void lw_neon_conv3x3_unit_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_NEON_CONV3X3 + const uint32_t height = (uint32_t)input_dimensions[2]; + const uint32_t width = (uint32_t)input_dimensions[3]; + const uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * + (uint32_t)input_dimensions[1] * 9u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + const float initial = bias == NULL ? 0.0f : bias[output_channel]; + const float32x4_t initial_values = vdupq_n_f32(initial); + uint64_t spatial = 0u; + uint32_t input_channel; + for (; spatial + 4u <= channel_plane; spatial += 4u) { + vst1q_f32(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + const uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + const uint32_t output_y_end = kernel_y == 2u ? height - 1u : height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + const uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + const uint32_t output_x_end = kernel_x == 2u ? width - 1u : width; + const float weight = weight_channel_data[kernel_y * 3u + kernel_x]; + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + const uint32_t input_y = output_y + kernel_y - 1u; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 4u <= output_x_end; output_x += 4u) { + const uint32_t input_x = output_x + kernel_x - 1u; + const float32x4_t input_values = vld1q_f32(input_row + input_x); + const float32x4_t output_values = + vld1q_f32(output_row + output_x); + vst1q_f32(output_row + output_x, + vaddq_f32(output_values, + vmulq_n_f32(input_values, weight))); + } + for (; output_x < output_x_end; ++output_x) { + const uint32_t input_x = output_x + kernel_x - 1u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } + } +#else + lw_scalar_conv3x3_unit_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/neon_convtranspose2x2.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/neon_convtranspose2x2.c new file mode 100644 index 00000000..5232a06c --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/neon_convtranspose2x2.c @@ -0,0 +1,122 @@ +#include "simd_kernels.h" + +#include "scalar_kernels.h" + +#include + +#if defined(_M_ARM64) || defined(__aarch64__) +# include +# define LW_COMPILES_NEON_CONV_TRANSPOSE2X2 1 +#else +# define LW_COMPILES_NEON_CONV_TRANSPOSE2X2 0 +#endif + +void lw_neon_conv_transpose2x2_stride2_range_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4], + uint32_t output_channel_begin, uint32_t output_channel_end) { +#if LW_COMPILES_NEON_CONV_TRANSPOSE2X2 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint32_t input_height = (uint32_t)input_dimensions[2]; + const uint32_t input_width = (uint32_t)input_dimensions[3]; + const uint32_t output_width = (uint32_t)output_dimensions[3]; + const uint64_t input_plane = (uint64_t)input_height * input_width; + const uint64_t output_plane = + (uint64_t)(uint32_t)output_dimensions[2] * output_width; + const float32x4_t zero = vdupq_n_f32(0.0f); + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * input_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * output_plane); + uint32_t output_channel; + for (output_channel = output_channel_begin; output_channel < output_channel_end; + ++output_channel) { + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * output_plane); + const float initial_value = bias == NULL ? 0.0f : bias[output_channel]; + const float32x4_t initial = vdupq_n_f32(initial_value); + uint64_t output_index = 0u; + uint32_t input_channel; + for (; output_index + 4u <= output_plane; output_index += 4u) { + vst1q_f32(output_channel_data + (size_t)output_index, initial); + } + for (; output_index < output_plane; ++output_index) { + output_channel_data[(size_t)output_index] = initial_value; + } + + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * input_plane); + const float* channel_weights = + weights + (size_t)(((uint64_t)input_channel * output_channels + + output_channel) * 4u); + const float weight0 = channel_weights[0]; + const float weight1 = channel_weights[1]; + const float weight2 = channel_weights[2]; + const float weight3 = channel_weights[3]; + uint32_t input_y; + for (input_y = 0u; input_y < input_height; ++input_y) { + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * input_width); + float* output_row0 = + output_channel_data + (size_t)((uint64_t)(input_y * 2u) * output_width); + float* output_row1 = output_row0 + output_width; + uint32_t input_x = 0u; + for (; input_x + 4u <= input_width; input_x += 4u) { + const float32x4_t values = vld1q_f32(input_row + input_x); + const float32x4x2_t even = vzipq_f32(values, zero); + const float32x4x2_t odd = vzipq_f32(zero, values); + const uint32_t output_x = input_x * 2u; + float32x4_t row0_low = vld1q_f32(output_row0 + output_x); + float32x4_t row0_high = vld1q_f32(output_row0 + output_x + 4u); + float32x4_t row1_low = vld1q_f32(output_row1 + output_x); + float32x4_t row1_high = vld1q_f32(output_row1 + output_x + 4u); + + row0_low = vaddq_f32(row0_low, vmulq_n_f32(even.val[0], weight0)); + row0_high = vaddq_f32(row0_high, vmulq_n_f32(even.val[1], weight0)); + row0_low = vaddq_f32(row0_low, vmulq_n_f32(odd.val[0], weight1)); + row0_high = vaddq_f32(row0_high, vmulq_n_f32(odd.val[1], weight1)); + row1_low = vaddq_f32(row1_low, vmulq_n_f32(even.val[0], weight2)); + row1_high = vaddq_f32(row1_high, vmulq_n_f32(even.val[1], weight2)); + row1_low = vaddq_f32(row1_low, vmulq_n_f32(odd.val[0], weight3)); + row1_high = vaddq_f32(row1_high, vmulq_n_f32(odd.val[1], weight3)); + + vst1q_f32(output_row0 + output_x, row0_low); + vst1q_f32(output_row0 + output_x + 4u, row0_high); + vst1q_f32(output_row1 + output_x, row1_low); + vst1q_f32(output_row1 + output_x + 4u, row1_high); + } + for (; input_x < input_width; ++input_x) { + const float value = input_row[input_x]; + const uint32_t output_x = input_x * 2u; + output_row0[output_x] += value * weight0; + output_row0[output_x + 1u] += value * weight1; + output_row1[output_x] += value * weight2; + output_row1[output_x + 1u] += value * weight3; + } + } + } + } + } +#else + (void)input; + (void)weights; + (void)bias; + (void)output; + (void)input_dimensions; + (void)output_dimensions; + (void)output_channel_begin; + (void)output_channel_end; +#endif +} + +void lw_neon_conv_transpose2x2_stride2_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { + lw_neon_conv_transpose2x2_stride2_range_f32( + input, weights, bias, output, input_dimensions, output_dimensions, 0u, + (uint32_t)output_dimensions[1]); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/neon_packed_conv1x1.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/neon_packed_conv1x1.c new file mode 100644 index 00000000..d10bb139 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/neon_packed_conv1x1.c @@ -0,0 +1,116 @@ +#include "simd_kernels.h" + +#include "packed_conv_internal.h" + +#include + +#if defined(_M_ARM64) || defined(__aarch64__) +# include +# define LW_COMPILES_NEON_PACKED_CONV1X1 1 +#else +# define LW_COMPILES_NEON_PACKED_CONV1X1 0 +#endif + +void lw_neon_packed_conv1x1_f32(const float* input, const float* packed_weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_NEON_PACKED_CONV1X1 + const uint32_t input_channels = (uint32_t)input_dimensions[1]; + const uint32_t output_channels = (uint32_t)output_dimensions[1]; + const uint64_t channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + const uint32_t output_blocks = + (output_channels + LW_PACKED_CONV1X1_OUTPUT_TILE - 1u) / + LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_block; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + const uint32_t output_base = output_block * LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t valid_outputs = output_channels - output_base; + uint64_t spatial = 0u; + if (valid_outputs > LW_PACKED_CONV1X1_OUTPUT_TILE) { + valid_outputs = LW_PACKED_CONV1X1_OUTPUT_TILE; + } + for (; spatial + 4u <= channel_plane; spatial += 4u) { + float32x4_t accumulator0 = + vdupq_n_f32(bias == NULL ? 0.0f : bias[output_base]); + float32x4_t accumulator1 = vdupq_n_f32( + bias == NULL || valid_outputs <= 1u ? 0.0f : bias[output_base + 1u]); + float32x4_t accumulator2 = vdupq_n_f32( + bias == NULL || valid_outputs <= 2u ? 0.0f : bias[output_base + 2u]); + float32x4_t accumulator3 = vdupq_n_f32( + bias == NULL || valid_outputs <= 3u ? 0.0f : bias[output_base + 3u]); + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + + (size_t)((uint64_t)input_channel * channel_plane + spatial); + const float* packed = + packed_weights + + (size_t)(((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE); + const float32x4_t input_values = vld1q_f32(input_channel_data); + accumulator0 = + vaddq_f32(accumulator0, vmulq_n_f32(input_values, packed[0])); + accumulator1 = + vaddq_f32(accumulator1, vmulq_n_f32(input_values, packed[1])); + accumulator2 = + vaddq_f32(accumulator2, vmulq_n_f32(input_values, packed[2])); + accumulator3 = + vaddq_f32(accumulator3, vmulq_n_f32(input_values, packed[3])); + } + vst1q_f32(batch_output + + (size_t)((uint64_t)output_base * channel_plane + spatial), + accumulator0); + if (valid_outputs > 1u) { + vst1q_f32(batch_output + + (size_t)((uint64_t)(output_base + 1u) * channel_plane + spatial), + accumulator1); + } + if (valid_outputs > 2u) { + vst1q_f32(batch_output + + (size_t)((uint64_t)(output_base + 2u) * channel_plane + spatial), + accumulator2); + } + if (valid_outputs > 3u) { + vst1q_f32(batch_output + + (size_t)((uint64_t)(output_base + 3u) * channel_plane + spatial), + accumulator3); + } + } + for (; spatial < channel_plane; ++spatial) { + float accumulators[LW_PACKED_CONV1X1_OUTPUT_TILE] = { + bias == NULL ? 0.0f : bias[output_base], + bias == NULL || valid_outputs <= 1u ? 0.0f : bias[output_base + 1u], + bias == NULL || valid_outputs <= 2u ? 0.0f : bias[output_base + 2u], + bias == NULL || valid_outputs <= 3u ? 0.0f : bias[output_base + 3u]}; + uint32_t input_channel; + uint32_t output_lane; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float input_value = + batch_input[(size_t)((uint64_t)input_channel * channel_plane + spatial)]; + const float* packed = + packed_weights + + (size_t)(((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE); + for (output_lane = 0u; output_lane < valid_outputs; ++output_lane) { + accumulators[output_lane] += input_value * packed[output_lane]; + } + } + for (output_lane = 0u; output_lane < valid_outputs; ++output_lane) { + batch_output[(size_t)(((uint64_t)output_base + output_lane) * channel_plane + + spatial)] = accumulators[output_lane]; + } + } + } + } +#else + lw_scalar_packed_conv1x1_f32(input, packed_weights, bias, output, input_dimensions, + output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/simd_kernels.h b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/simd_kernels.h new file mode 100644 index 00000000..9ddc73e2 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/simd_kernels.h @@ -0,0 +1,217 @@ +#ifndef LW_SIMD_KERNELS_H +#define LW_SIMD_KERNELS_H + +/* Architecture-specific kernels; use only after checking lw_cpu_simd_level(). */ + +#include "scalar_kernels.h" + +#include + +void lw_sse2_binary_contiguous_f32(lw_scalar_binary_op operation, const float* left, + const float* right, float* output, uint64_t element_count); +void lw_sse2_binary_right_scalar_f32(lw_scalar_binary_op operation, const float* left, float right, + float* output, uint64_t element_count); +void lw_avx2_binary_contiguous_f32(lw_scalar_binary_op operation, const float* left, + const float* right, float* output, uint64_t element_count); +void lw_avx2_binary_right_scalar_f32(lw_scalar_binary_op operation, const float* left, float right, + float* output, uint64_t element_count); +void lw_avx2_relu_contiguous_f32(const float* input, float* output, uint64_t element_count); +void lw_avx2_binary_channel_f32(lw_scalar_binary_op operation, const float* full, const float* channel, float* output, uint64_t pixels, uint32_t channels, int broadcast_is_left); +void lw_avx2_binary_channel_nchw_f32(lw_scalar_binary_op operation, const float* full, const float* channel, float* output, uint64_t spatial, uint32_t channels, int broadcast_is_left); +void lw_avx2_hard_sigmoid_contiguous_f32(const float* input, float* output, uint64_t element_count, float alpha, float beta); +void lw_avx2_hard_sigmoid_exact_f32(const float* input, float* output, uint64_t element_count, float alpha, float beta); +void lw_avx2_relu_f32(const float* input, float* output, uint64_t element_count); +void lw_avx2_erf_f32(const float* input, float* output, uint64_t element_count); +void lw_avx2_gelu_f32(const float* input, float* output, uint64_t element_count); +/* Transpose a rows x cols row-major float matrix into cols x rows row-major. + * Pure data movement, bit-identical to the scalar transpose. */ +void lw_avx2_transpose_2d_f32(const float* input, float* output, + uint32_t rows, uint32_t cols); +void lw_avx2_softmax_contiguous_f32(const float* input, float* output, uint64_t row_count, + uint64_t axis_count); +void lw_avx2_ctc_emitted_softmax_contiguous_f32(const float* input, + const uint32_t* best_indices, + float* emitted_probabilities, + uint64_t row_count, uint64_t axis_count); +void lw_wasm128_erf_f32(const float* input, float* output, uint64_t element_count); +void lw_wasm128_gelu_f32(const float* input, float* output, uint64_t element_count); +void lw_wasm128_binary_contiguous_f32(lw_scalar_binary_op operation, + const float* left, const float* right, + float* output, uint64_t count); +void lw_wasm128_binary_scalar_f32(lw_scalar_binary_op operation, + const float* tensor, float scalar, float* output, + uint64_t count, int scalar_is_left); +void lw_wasm128_binary_channel_nhwc_f32(lw_scalar_binary_op operation, + const float* tensor, const float* channel, float* output, + uint64_t pixels, uint32_t channels, int channel_is_left); +void lw_wasm128_affine_nhwc_f32(const float* input, const float* mul, + const float* add, float* output, uint32_t pixels, uint32_t channels); +void lw_wasm128_relu_f32(const float* input, float* output, uint64_t count); +int lw_wasm128_threshold_bitmap_f32(const float* prediction, uint8_t* bitmap, + uint64_t pixel_count, float threshold); +void lw_wasm128_reduce_mean_hw_f32(const float* input, float* output, + uint32_t batch, uint32_t height, uint32_t width, uint32_t channels); +void lw_wasm128_pool_nhwc_f32(const float* input, float* output, + uint32_t batch, uint32_t input_height, uint32_t input_width, + uint32_t output_height, uint32_t output_width, uint32_t channels, + uint32_t kernel_h, uint32_t kernel_w, uint32_t stride_h, uint32_t stride_w, + uint32_t pad_top, uint32_t pad_left, uint8_t count_include_pad, uint8_t is_max); +void lw_wasm128_packed_matmul_shared_f32(const float* input, + const float* packed_weights, float* output, uint32_t batch_count, + uint32_t rows, uint32_t inner_dimension, uint32_t columns); + +void lw_sse2_matmul_shared_f32(const float* input, const float* weights, float* output, + uint32_t batch_count, uint32_t rows, uint32_t inner_dimension, + uint32_t columns); +void lw_avx2_matmul_shared_f32(const float* input, const float* weights, float* output, + uint32_t batch_count, uint32_t rows, uint32_t inner_dimension, + uint32_t columns); +void lw_avx2_packed_matmul_shared_f32(const float* input, const float* packed_weights, + float* output, uint32_t batch_count, uint32_t rows, + uint32_t inner_dimension, uint32_t columns); +void lw_avx2_packed_matmul_bias_argmax_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + uint32_t* best_indices, uint32_t batch_count, uint32_t rows, + uint32_t inner_dimension, uint32_t columns); +void lw_avx2_fma_packed_matmul_bias_argmax_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + uint32_t* best_indices, uint32_t batch_count, uint32_t rows, + uint32_t inner_dimension, uint32_t columns); +void lw_avx2_fma_packed_matmul_argmax_scores_f32( + const float* input, const float* packed_weights, const float* bias, + uint32_t* best_indices, float* scores, uint32_t batch_count, uint32_t rows, + uint32_t inner_dimension, uint32_t columns); +void lw_avx2_ctc_row_probabilities_f32( + const float* input, const float* packed_weights, const float* bias, + const uint32_t* best_indices, const float* scores, float* probabilities, + uint32_t rows, uint32_t inner_dimension, uint32_t columns); +int lw_avx2_threshold_bitmap_f32(const float* prediction, uint8_t* bitmap, + uint64_t pixel_count, float threshold); +void lw_avx2_interior_bitmap_u8(const uint8_t* bitmap, uint8_t* interior, + uint32_t width, uint32_t height); +void lw_sse2_conv1x1_unit_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4], uint32_t groups, + uint32_t input_channels_per_group, + uint32_t output_channels_per_group); +void lw_avx2_conv1x1_unit_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4], uint32_t groups, + uint32_t input_channels_per_group, + uint32_t output_channels_per_group); +void lw_sse2_packed_conv1x1_f32(const float* input, const float* packed_weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_fma_packed_conv1x1_f32(const float* input, const float* packed_weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_fma_packed_conv1x1_8x8_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); +void lw_avx2_packed_conv1x1_f32(const float* input, const float* packed_weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_neon_packed_conv1x1_f32(const float* input, const float* packed_weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_lsx_packed_conv1x1_f32(const float* input, const float* packed_weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_sse2_depthwise_conv3x3_unit_pad1_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]); +void lw_avx2_depthwise_conv3x3_unit_pad1_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]); +void lw_sse2_depthwise_conv3x3_stride2x1_pad1_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); +void lw_avx2_depthwise_conv3x3_stride2x1_pad1_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); +void lw_sse2_depthwise_conv5x5_unit_pad2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]); +void lw_avx2_depthwise_conv5x5_unit_pad2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]); +void lw_avx2_depthwise_conv9x9_unit_pad4_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]); +void lw_sse2_conv3x3_unit_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_conv3x3_unit_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_conv7x7_unit_pad3_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_conv7x7_four_outputs_unit_pad3_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); +void lw_avx2_conv5x5_unit_pad2_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_conv5x5_four_outputs_unit_pad2_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); +void lw_avx2_conv7x1_unit_pad3_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_conv1x7_unit_pad3_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_conv5x1_unit_pad2_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_conv1x5_unit_pad2_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_neon_conv3x3_unit_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_sse2_conv2x2_unit_pad_end1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_conv2x2_unit_pad_end1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_sse2_conv3x3_stride2_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_conv3x3_stride2_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_packed_conv3x3_stride2_pad1_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); +void lw_avx2_fma_packed_conv3x3_stride2_pad1_f32( + const float* input, const float* packed_weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]); +void lw_avx2_conv_transpose2x2_stride2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_avx2_conv_transpose2x2_stride2_range_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4], + uint32_t output_channel_begin, uint32_t output_channel_end); +void lw_sse2_conv_transpose2x2_stride2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_sse2_conv_transpose2x2_stride2_range_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4], + uint32_t output_channel_begin, uint32_t output_channel_end); +void lw_neon_conv_transpose2x2_stride2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]); +void lw_neon_conv_transpose2x2_stride2_range_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4], + uint32_t output_channel_begin, uint32_t output_channel_end); + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/simd_platform.h b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/simd_platform.h new file mode 100644 index 00000000..1bd1fe31 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/simd_platform.h @@ -0,0 +1,27 @@ +#ifndef LW_SIMD_PLATFORM_H +#define LW_SIMD_PLATFORM_H + +/* + * The Emscripten toolchain translates the portable SSE2 intrinsic subset to + * WebAssembly SIMD128 when the translation unit is compiled with + * -msimd128. Keeping this small compatibility layer lets the existing SSE2 + * kernels serve as the first WASM SIMD backend without changing their + * numerical algorithms or the public runtime ABI. + */ +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) || \ + (defined(__EMSCRIPTEN__) && defined(__wasm_simd128__)) +# include +# define LW_SIMD_HAS_SSE2_INTRINSICS 1 +#else +# define LW_SIMD_HAS_SSE2_INTRINSICS 0 +#endif + +/* target("sse2") is meaningful only for native x86 Clang/GCC builds. */ +#if LW_SIMD_HAS_SSE2_INTRINSICS && !defined(__EMSCRIPTEN__) && \ + (defined(__GNUC__) || defined(__clang__)) +# define LW_SIMD_SSE2_TARGET __attribute__((target("sse2"))) +#else +# define LW_SIMD_SSE2_TARGET +#endif + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_binary.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_binary.c new file mode 100644 index 00000000..eb952a78 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_binary.c @@ -0,0 +1,70 @@ +#include "simd_kernels.h" +#include "simd_platform.h" + +#include + +#define LW_COMPILES_SSE2_BINARY LW_SIMD_HAS_SSE2_INTRINSICS + +LW_SIMD_SSE2_TARGET +void lw_sse2_binary_contiguous_f32( + lw_scalar_binary_op operation, + const float* left, + const float* right, + float* output, + uint64_t element_count) { + uint64_t index = 0u; +#if LW_COMPILES_SSE2_BINARY + if (operation == LW_SCALAR_BINARY_ADD) { + for (; index + 4u <= element_count; index += 4u) { + __m128 result = + _mm_add_ps(_mm_loadu_ps(left + (size_t)index), _mm_loadu_ps(right + (size_t)index)); + _mm_storeu_ps(output + (size_t)index, result); + } + } else if (operation == LW_SCALAR_BINARY_MUL) { + for (; index + 4u <= element_count; index += 4u) { + __m128 result = + _mm_mul_ps(_mm_loadu_ps(left + (size_t)index), _mm_loadu_ps(right + (size_t)index)); + _mm_storeu_ps(output + (size_t)index, result); + } + } else if (operation == LW_SCALAR_BINARY_DIV) { + for (; index + 4u <= element_count; index += 4u) { + __m128 result = + _mm_div_ps(_mm_loadu_ps(left + (size_t)index), _mm_loadu_ps(right + (size_t)index)); + _mm_storeu_ps(output + (size_t)index, result); + } + } +#endif + lw_scalar_binary_contiguous_f32(operation, left + (size_t)index, right + (size_t)index, + output + (size_t)index, element_count - index); +} + +LW_SIMD_SSE2_TARGET +void lw_sse2_binary_right_scalar_f32( + lw_scalar_binary_op operation, + const float* left, + float right, + float* output, + uint64_t element_count) { + uint64_t index = 0u; +#if LW_COMPILES_SSE2_BINARY + __m128 right_values = _mm_set1_ps(right); + if (operation == LW_SCALAR_BINARY_ADD) { + for (; index + 4u <= element_count; index += 4u) { + __m128 result = _mm_add_ps(_mm_loadu_ps(left + (size_t)index), right_values); + _mm_storeu_ps(output + (size_t)index, result); + } + } else if (operation == LW_SCALAR_BINARY_MUL) { + for (; index + 4u <= element_count; index += 4u) { + __m128 result = _mm_mul_ps(_mm_loadu_ps(left + (size_t)index), right_values); + _mm_storeu_ps(output + (size_t)index, result); + } + } else if (operation == LW_SCALAR_BINARY_DIV) { + for (; index + 4u <= element_count; index += 4u) { + __m128 result = _mm_div_ps(_mm_loadu_ps(left + (size_t)index), right_values); + _mm_storeu_ps(output + (size_t)index, result); + } + } +#endif + lw_scalar_binary_right_scalar_f32(operation, left + (size_t)index, right, + output + (size_t)index, element_count - index); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_conv1x1.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_conv1x1.c new file mode 100644 index 00000000..88ce5a7f --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_conv1x1.c @@ -0,0 +1,83 @@ +#include "simd_kernels.h" +#include "simd_platform.h" + +#include "scalar_kernels.h" + +#include + +#define LW_COMPILES_SSE2_CONV1X1 LW_SIMD_HAS_SSE2_INTRINSICS + +LW_SIMD_SSE2_TARGET +void lw_sse2_conv1x1_unit_f32( + const float* input, + const float* weights, + const float* bias, + float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4], + uint32_t groups, + uint32_t input_channels_per_group, + uint32_t output_channels_per_group) { +#if LW_COMPILES_SSE2_CONV1X1 + uint64_t channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + uint32_t group; + for (group = 0u; group < groups; ++group) { + uint32_t input_channel_start = group * input_channels_per_group; + uint32_t output_channel_start = group * output_channels_per_group; + const float* group_input = + input + + (size_t)(((uint64_t)batch * (uint32_t)input_dimensions[1] + input_channel_start) * + channel_plane); + float* group_output = + output + + (size_t)(((uint64_t)batch * (uint32_t)output_dimensions[1] + output_channel_start) * + channel_plane); + uint32_t output_channel_in_group; + for (output_channel_in_group = 0u; output_channel_in_group < output_channels_per_group; + ++output_channel_in_group) { + uint32_t output_channel = output_channel_start + output_channel_in_group; + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * input_channels_per_group); + float* output_channel_data = + group_output + (size_t)((uint64_t)output_channel_in_group * channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + __m128 initial_values = _mm_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel_in_group; + for (; spatial + 4u <= channel_plane; spatial += 4u) { + _mm_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel_in_group = 0u; input_channel_in_group < input_channels_per_group; + ++input_channel_in_group) { + const float* input_channel_data = + group_input + (size_t)((uint64_t)input_channel_in_group * channel_plane); + __m128 weight_values = + _mm_set1_ps(output_channel_weights[input_channel_in_group]); + spatial = 0u; + for (; spatial + 4u <= channel_plane; spatial += 4u) { + __m128 output_values = _mm_loadu_ps(output_channel_data + (size_t)spatial); + __m128 input_values = _mm_loadu_ps(input_channel_data + (size_t)spatial); + output_values = + _mm_add_ps(output_values, _mm_mul_ps(input_values, weight_values)); + _mm_storeu_ps(output_channel_data + (size_t)spatial, output_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] += + input_channel_data[(size_t)spatial] * + output_channel_weights[input_channel_in_group]; + } + } + } + } + } +#else + lw_scalar_conv1x1_unit_f32(input, weights, bias, output, input_dimensions, output_dimensions, + groups, input_channels_per_group, output_channels_per_group); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_conv2x2.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_conv2x2.c new file mode 100644 index 00000000..10e5c2fa --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_conv2x2.c @@ -0,0 +1,197 @@ +#include "simd_kernels.h" +#include "simd_platform.h" + +#include "scalar_kernels.h" + +#include + +#define LW_COMPILES_SSE2_CONV2X2 LW_SIMD_HAS_SSE2_INTRINSICS + +#if LW_COMPILES_SSE2_CONV2X2 +LW_SIMD_SSE2_TARGET +static void conv2x2_unit_pad_end1_four_outputs_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { + uint32_t height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + output_channel += 4u) { + const uint64_t weights_per_output = (uint64_t)(uint32_t)input_dimensions[1] * 4u; + const float* weights0 = weights + (size_t)((uint64_t)(output_channel + 0u) * weights_per_output); + const float* weights1 = weights + (size_t)((uint64_t)(output_channel + 1u) * weights_per_output); + const float* weights2 = weights + (size_t)((uint64_t)(output_channel + 2u) * weights_per_output); + const float* weights3 = weights + (size_t)((uint64_t)(output_channel + 3u) * weights_per_output); + float* output0 = batch_output + (size_t)((uint64_t)(output_channel + 0u) * channel_plane); + float* output1 = batch_output + (size_t)((uint64_t)(output_channel + 1u) * channel_plane); + float* output2 = batch_output + (size_t)((uint64_t)(output_channel + 2u) * channel_plane); + float* output3 = batch_output + (size_t)((uint64_t)(output_channel + 3u) * channel_plane); + const float initial0 = bias == NULL ? 0.0f : bias[output_channel + 0u]; + const float initial1 = bias == NULL ? 0.0f : bias[output_channel + 1u]; + const float initial2 = bias == NULL ? 0.0f : bias[output_channel + 2u]; + const float initial3 = bias == NULL ? 0.0f : bias[output_channel + 3u]; + __m128 initial_values0 = _mm_set1_ps(initial0); + __m128 initial_values1 = _mm_set1_ps(initial1); + __m128 initial_values2 = _mm_set1_ps(initial2); + __m128 initial_values3 = _mm_set1_ps(initial3); + uint64_t spatial = 0u; + for (; spatial + 4u <= channel_plane; spatial += 4u) { + _mm_storeu_ps(output0 + (size_t)spatial, initial_values0); + _mm_storeu_ps(output1 + (size_t)spatial, initial_values1); + _mm_storeu_ps(output2 + (size_t)spatial, initial_values2); + _mm_storeu_ps(output3 + (size_t)spatial, initial_values3); + } + for (; spatial < channel_plane; ++spatial) { + output0[(size_t)spatial] = initial0; + output1[(size_t)spatial] = initial1; + output2[(size_t)spatial] = initial2; + output3[(size_t)spatial] = initial3; + } + uint32_t input_channel; + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weights0_channel = weights0 + (size_t)input_channel * 4u; + const float* weights1_channel = weights1 + (size_t)input_channel * 4u; + const float* weights2_channel = weights2 + (size_t)input_channel * 4u; + const float* weights3_channel = weights3 + (size_t)input_channel * 4u; + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 2u; ++kernel_y) { + uint32_t output_y; + uint32_t output_y_end = height - kernel_y; + for (output_y = 0u; output_y < output_y_end; ++output_y) { + const float* input_row = input_channel_data + + (size_t)((uint64_t)(output_y + kernel_y) * width); + float* output_row0 = output0 + (size_t)((uint64_t)output_y * width); + float* output_row1 = output1 + (size_t)((uint64_t)output_y * width); + float* output_row2 = output2 + (size_t)((uint64_t)output_y * width); + float* output_row3 = output3 + (size_t)((uint64_t)output_y * width); + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 2u; ++kernel_x) { + uint32_t output_x = 0u; + uint32_t output_x_end = width - kernel_x; + __m128 weight0 = _mm_set1_ps(weights0_channel[kernel_y * 2u + kernel_x]); + __m128 weight1 = _mm_set1_ps(weights1_channel[kernel_y * 2u + kernel_x]); + __m128 weight2 = _mm_set1_ps(weights2_channel[kernel_y * 2u + kernel_x]); + __m128 weight3 = _mm_set1_ps(weights3_channel[kernel_y * 2u + kernel_x]); + const float* input_values = input_row + kernel_x; + for (; output_x + 4u <= output_x_end; output_x += 4u) { + __m128 values = _mm_loadu_ps(input_values + output_x); + __m128 values0 = _mm_loadu_ps(output_row0 + output_x); + __m128 values1 = _mm_loadu_ps(output_row1 + output_x); + __m128 values2 = _mm_loadu_ps(output_row2 + output_x); + __m128 values3 = _mm_loadu_ps(output_row3 + output_x); + _mm_storeu_ps(output_row0 + output_x, + _mm_add_ps(values0, _mm_mul_ps(values, weight0))); + _mm_storeu_ps(output_row1 + output_x, + _mm_add_ps(values1, _mm_mul_ps(values, weight1))); + _mm_storeu_ps(output_row2 + output_x, + _mm_add_ps(values2, _mm_mul_ps(values, weight2))); + _mm_storeu_ps(output_row3 + output_x, + _mm_add_ps(values3, _mm_mul_ps(values, weight3))); + } + for (; output_x < output_x_end; ++output_x) { + float value = input_values[output_x]; + output_row0[output_x] += value * weights0_channel[kernel_y * 2u + kernel_x]; + output_row1[output_x] += value * weights1_channel[kernel_y * 2u + kernel_x]; + output_row2[output_x] += value * weights2_channel[kernel_y * 2u + kernel_x]; + output_row3[output_x] += value * weights3_channel[kernel_y * 2u + kernel_x]; + } + } + } + } + } + } + } +} +#endif + +LW_SIMD_SSE2_TARGET +void lw_sse2_conv2x2_unit_pad_end1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_SSE2_CONV2X2 + if ((uint32_t)output_dimensions[1] % 4u == 0u) { + conv2x2_unit_pad_end1_four_outputs_f32(input, weights, bias, output, input_dimensions, + output_dimensions); + return; + } + uint32_t height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * (uint32_t)input_dimensions[1] * 4u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + __m128 initial_values = _mm_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel; + for (; spatial + 4u <= channel_plane; spatial += 4u) { + _mm_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 4u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 2u; ++kernel_y) { + uint32_t output_y_end = height - kernel_y; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 2u; ++kernel_x) { + uint32_t output_x_end = width - kernel_x; + __m128 weight_values = + _mm_set1_ps(weight_channel_data[kernel_y * 2u + kernel_x]); + uint32_t output_y; + for (output_y = 0u; output_y < output_y_end; ++output_y) { + const float* input_row = + input_channel_data + + (size_t)((uint64_t)(output_y + kernel_y) * width + kernel_x); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x = 0u; + for (; output_x + 4u <= output_x_end; output_x += 4u) { + __m128 input_values = _mm_loadu_ps(input_row + output_x); + __m128 output_values = _mm_loadu_ps(output_row + output_x); + output_values = _mm_add_ps(output_values, + _mm_mul_ps(input_values, weight_values)); + _mm_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + output_row[output_x] += + input_row[output_x] * + weight_channel_data[kernel_y * 2u + kernel_x]; + } + } + } + } + } + } + } +#else + lw_scalar_conv2x2_unit_pad_end1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_conv3x3.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_conv3x3.c new file mode 100644 index 00000000..87f95fbf --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_conv3x3.c @@ -0,0 +1,190 @@ +#include "simd_kernels.h" +#include "simd_platform.h" + +#include "scalar_kernels.h" + +#include + +#define LW_COMPILES_SSE2_CONV3X3 LW_SIMD_HAS_SSE2_INTRINSICS + +LW_SIMD_SSE2_TARGET +void lw_sse2_conv3x3_unit_pad1_f32(const float* input, const float* weights, const float* bias, + float* output, const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_SSE2_CONV3X3 + uint32_t height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * channel_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * (uint32_t)input_dimensions[1] * 9u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + __m128 initial_values = _mm_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel; + for (; spatial + 4u <= channel_plane; spatial += 4u) { + _mm_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = kernel_y == 2u ? height - 1u : height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = kernel_x == 2u ? width - 1u : width; + float weight = weight_channel_data[kernel_y * 3u + kernel_x]; + __m128 weight_values = _mm_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y + kernel_y - 1u; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * width); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 4u <= output_x_end; output_x += 4u) { + uint32_t input_x = output_x + kernel_x - 1u; + __m128 input_values = _mm_loadu_ps(input_row + input_x); + __m128 output_values = _mm_loadu_ps(output_row + output_x); + output_values = _mm_add_ps(output_values, + _mm_mul_ps(input_values, weight_values)); + _mm_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 1u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } + } +#else + lw_scalar_conv3x3_unit_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); +#endif +} + +#if LW_COMPILES_SSE2_CONV3X3 && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("sse2"))) +#endif +void lw_sse2_conv3x3_stride2_pad1_f32( + const float* input, + const float* weights, + const float* bias, + float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_SSE2_CONV3X3 + uint32_t input_height = (uint32_t)input_dimensions[2]; + uint32_t input_width = (uint32_t)input_dimensions[3]; + uint32_t output_height = (uint32_t)output_dimensions[2]; + uint32_t output_width = (uint32_t)output_dimensions[3]; + uint64_t input_channel_plane = (uint64_t)input_height * input_width; + uint64_t output_channel_plane = (uint64_t)output_height * output_width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * (uint32_t)input_dimensions[1] * input_channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * (uint32_t)output_dimensions[1] * + output_channel_plane); + uint32_t output_channel; + for (output_channel = 0u; output_channel < (uint32_t)output_dimensions[1]; + ++output_channel) { + const float* output_channel_weights = + weights + (size_t)((uint64_t)output_channel * (uint32_t)input_dimensions[1] * 9u); + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * output_channel_plane); + float initial = bias == NULL ? 0.0f : bias[output_channel]; + __m128 initial_values = _mm_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t input_channel; + for (; spatial + 4u <= output_channel_plane; spatial += 4u) { + _mm_storeu_ps(output_channel_data + (size_t)spatial, initial_values); + } + for (; spatial < output_channel_plane; ++spatial) { + output_channel_data[(size_t)spatial] = initial; + } + for (input_channel = 0u; input_channel < (uint32_t)input_dimensions[1]; + ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * input_channel_plane); + const float* weight_channel_data = + output_channel_weights + (size_t)((uint64_t)input_channel * 9u); + uint32_t kernel_y; + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = output_height; + uint32_t kernel_x; + if (kernel_y == 2u && (uint64_t)(output_y_end - 1u) * 2u + 1u >= input_height) { + --output_y_end; + } + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = output_width; + float weight = weight_channel_data[(size_t)kernel_y * 3u + kernel_x]; + __m128 weight_values = _mm_set1_ps(weight); + uint32_t output_y; + if (kernel_x == 2u && + (uint64_t)(output_x_end - 1u) * 2u + 1u >= input_width) { + --output_x_end; + } + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y * 2u - 1u + kernel_y; + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * input_width); + float* output_row = + output_channel_data + (size_t)((uint64_t)output_y * output_width); + uint32_t output_x = output_x_begin; + for (; output_x + 4u <= output_x_end; output_x += 4u) { + uint32_t input_x = output_x * 2u - 1u + kernel_x; + if ((uint64_t)input_x + 7u >= input_width) { + break; + } + { + __m128 first = _mm_loadu_ps(input_row + input_x); + __m128 second = _mm_loadu_ps(input_row + input_x + 4u); + __m128 input_values = + _mm_shuffle_ps(first, second, _MM_SHUFFLE(2, 0, 2, 0)); + __m128 output_values = _mm_loadu_ps(output_row + output_x); + output_values = _mm_add_ps( + output_values, _mm_mul_ps(input_values, weight_values)); + _mm_storeu_ps(output_row + output_x, output_values); + } + } + for (; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x * 2u - 1u + kernel_x; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } + } +#else + lw_scalar_conv3x3_stride2_pad1_f32(input, weights, bias, output, input_dimensions, + output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_convtranspose2x2.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_convtranspose2x2.c new file mode 100644 index 00000000..0e26468d --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_convtranspose2x2.c @@ -0,0 +1,130 @@ +#include "simd_kernels.h" +#include "simd_platform.h" + +#include "scalar_kernels.h" + +#include + +#define LW_COMPILES_SSE2_CONV_TRANSPOSE2X2 LW_SIMD_HAS_SSE2_INTRINSICS + +LW_SIMD_SSE2_TARGET +void lw_sse2_conv_transpose2x2_stride2_range_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4], + uint32_t output_channel_begin, uint32_t output_channel_end) { +#if LW_COMPILES_SSE2_CONV_TRANSPOSE2X2 + uint32_t input_channels = (uint32_t)input_dimensions[1]; + uint32_t output_channels = (uint32_t)output_dimensions[1]; + uint32_t input_height = (uint32_t)input_dimensions[2]; + uint32_t input_width = (uint32_t)input_dimensions[3]; + uint32_t output_width = (uint32_t)output_dimensions[3]; + uint64_t input_plane = (uint64_t)input_height * input_width; + uint64_t output_plane = (uint64_t)(uint32_t)output_dimensions[2] * output_width; + uint32_t batch; + + /* The AVX2 implementation handles eight values at once. This SSE2 + * counterpart keeps the identical no-overlap 2x2 expansion strategy for + * CPUs that expose SSE2 but not AVX2. */ + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * input_plane); + float* batch_output = + output + (size_t)((uint64_t)batch * output_channels * output_plane); + uint32_t output_channel; + for (output_channel = output_channel_begin; output_channel < output_channel_end; + ++output_channel) { + float* output_channel_data = + batch_output + (size_t)((uint64_t)output_channel * output_plane); + float initial_value = bias == NULL ? 0.0f : bias[output_channel]; + __m128 initial = _mm_set1_ps(initial_value); + uint64_t output_index = 0u; + uint32_t input_channel; + for (; output_index + 4u <= output_plane; output_index += 4u) { + _mm_storeu_ps(output_channel_data + (size_t)output_index, initial); + } + for (; output_index < output_plane; ++output_index) { + output_channel_data[(size_t)output_index] = initial_value; + } + + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * input_plane); + const float* channel_weights = + weights + (size_t)(((uint64_t)input_channel * output_channels + output_channel) * 4u); + uint32_t input_y; + for (input_y = 0u; input_y < input_height; ++input_y) { + const float* input_row = + input_channel_data + (size_t)((uint64_t)input_y * input_width); + float* output_row0 = + output_channel_data + (size_t)((uint64_t)(input_y * 2u) * output_width); + float* output_row1 = output_row0 + output_width; + uint32_t input_x = 0u; + for (; input_x + 4u <= input_width; input_x += 4u) { + __m128 values = _mm_loadu_ps(input_row + input_x); + __m128 zero = _mm_setzero_ps(); + __m128 even = _mm_unpacklo_ps(values, zero); + __m128 even_high = _mm_unpackhi_ps(values, zero); + __m128 odd = _mm_unpacklo_ps(zero, values); + __m128 odd_high = _mm_unpackhi_ps(zero, values); + uint32_t output_x = input_x * 2u; + __m128 product; + + product = _mm_mul_ps(even, _mm_set1_ps(channel_weights[0])); + _mm_storeu_ps(output_row0 + output_x, + _mm_add_ps(_mm_loadu_ps(output_row0 + output_x), product)); + product = _mm_mul_ps(even_high, _mm_set1_ps(channel_weights[0])); + _mm_storeu_ps(output_row0 + output_x + 4u, + _mm_add_ps(_mm_loadu_ps(output_row0 + output_x + 4u), product)); + + product = _mm_mul_ps(odd, _mm_set1_ps(channel_weights[1])); + _mm_storeu_ps(output_row0 + output_x, + _mm_add_ps(_mm_loadu_ps(output_row0 + output_x), product)); + product = _mm_mul_ps(odd_high, _mm_set1_ps(channel_weights[1])); + _mm_storeu_ps(output_row0 + output_x + 4u, + _mm_add_ps(_mm_loadu_ps(output_row0 + output_x + 4u), product)); + + product = _mm_mul_ps(even, _mm_set1_ps(channel_weights[2])); + _mm_storeu_ps(output_row1 + output_x, + _mm_add_ps(_mm_loadu_ps(output_row1 + output_x), product)); + product = _mm_mul_ps(even_high, _mm_set1_ps(channel_weights[2])); + _mm_storeu_ps(output_row1 + output_x + 4u, + _mm_add_ps(_mm_loadu_ps(output_row1 + output_x + 4u), product)); + + product = _mm_mul_ps(odd, _mm_set1_ps(channel_weights[3])); + _mm_storeu_ps(output_row1 + output_x, + _mm_add_ps(_mm_loadu_ps(output_row1 + output_x), product)); + product = _mm_mul_ps(odd_high, _mm_set1_ps(channel_weights[3])); + _mm_storeu_ps(output_row1 + output_x + 4u, + _mm_add_ps(_mm_loadu_ps(output_row1 + output_x + 4u), product)); + } + for (; input_x < input_width; ++input_x) { + float value = input_row[input_x]; + uint32_t output_x = input_x * 2u; + output_row0[output_x] += value * channel_weights[0]; + output_row0[output_x + 1u] += value * channel_weights[1]; + output_row1[output_x] += value * channel_weights[2]; + output_row1[output_x + 1u] += value * channel_weights[3]; + } + } + } + } + } +#else + (void)input; + (void)weights; + (void)bias; + (void)output; + (void)input_dimensions; + (void)output_dimensions; + (void)output_channel_begin; + (void)output_channel_end; +#endif +} + +void lw_sse2_conv_transpose2x2_stride2_f32( + const float* input, const float* weights, const float* bias, float* output, + const int32_t input_dimensions[4], const int32_t output_dimensions[4]) { + lw_sse2_conv_transpose2x2_stride2_range_f32( + input, weights, bias, output, input_dimensions, output_dimensions, 0u, + (uint32_t)output_dimensions[1]); +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_depthwise3x3.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_depthwise3x3.c new file mode 100644 index 00000000..6de7c8e6 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_depthwise3x3.c @@ -0,0 +1,149 @@ +#include "simd_kernels.h" +#include "simd_platform.h" + +#include + +#define LW_COMPILES_SSE2_DEPTHWISE3X3 LW_SIMD_HAS_SSE2_INTRINSICS + +LW_SIMD_SSE2_TARGET +void lw_sse2_depthwise_conv3x3_unit_pad1_f32( + const float* input, + const float* weights, + const float* bias, + float* output, + const int32_t dimensions[4]) { +#if LW_COMPILES_SSE2_DEPTHWISE3X3 + uint32_t height = (uint32_t)dimensions[2]; + uint32_t width = (uint32_t)dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)dimensions[0]; ++batch) { + uint32_t channel; + for (channel = 0u; channel < (uint32_t)dimensions[1]; ++channel) { + uint64_t channel_offset = + ((uint64_t)batch * (uint32_t)dimensions[1] + channel) * channel_plane; + const float* input_channel = input + (size_t)channel_offset; + const float* channel_weights = weights + (size_t)channel * 9u; + float* output_channel = output + (size_t)channel_offset; + float initial = bias == NULL ? 0.0f : bias[channel]; + __m128 initial_values = _mm_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t kernel_y; + for (; spatial + 4u <= channel_plane; spatial += 4u) { + _mm_storeu_ps(output_channel + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel[(size_t)spatial] = initial; + } + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = kernel_y == 2u && height != 0u ? height - 1u : height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = kernel_x == 2u && width != 0u ? width - 1u : width; + float weight = channel_weights[kernel_y * 3u + kernel_x]; + __m128 weight_values = _mm_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y + kernel_y - 1u; + const float* input_row = + input_channel + (size_t)((uint64_t)input_y * width); + float* output_row = output_channel + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 4u <= output_x_end; output_x += 4u) { + uint32_t input_x = output_x + kernel_x - 1u; + __m128 output_values = _mm_loadu_ps(output_row + output_x); + __m128 input_values = _mm_loadu_ps(input_row + input_x); + output_values = + _mm_add_ps(output_values, _mm_mul_ps(input_values, weight_values)); + _mm_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 1u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } +#else + lw_scalar_depthwise_conv3x3_unit_pad1_f32(input, weights, bias, output, dimensions); +#endif +} + +LW_SIMD_SSE2_TARGET +void lw_sse2_depthwise_conv3x3_stride2x1_pad1_f32( + const float* input, + const float* weights, + const float* bias, + float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_SSE2_DEPTHWISE3X3 + uint32_t input_height = (uint32_t)input_dimensions[2]; + uint32_t width = (uint32_t)input_dimensions[3]; + uint32_t output_height = (uint32_t)output_dimensions[2]; + uint64_t input_channel_plane = (uint64_t)input_height * width; + uint64_t output_channel_plane = (uint64_t)output_height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + uint32_t channel; + for (channel = 0u; channel < (uint32_t)input_dimensions[1]; ++channel) { + uint64_t input_channel_offset = + ((uint64_t)batch * (uint32_t)input_dimensions[1] + channel) * input_channel_plane; + uint64_t output_channel_offset = + ((uint64_t)batch * (uint32_t)output_dimensions[1] + channel) * + output_channel_plane; + const float* input_channel = input + (size_t)input_channel_offset; + const float* channel_weights = weights + (size_t)channel * 9u; + float* output_channel = output + (size_t)output_channel_offset; + float initial = bias == NULL ? 0.0f : bias[channel]; + __m128 initial_values = _mm_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t kernel_y; + for (; spatial + 4u <= output_channel_plane; spatial += 4u) { + _mm_storeu_ps(output_channel + (size_t)spatial, initial_values); + } + for (; spatial < output_channel_plane; ++spatial) { + output_channel[(size_t)spatial] = initial; + } + for (kernel_y = 0u; kernel_y < 3u; ++kernel_y) { + uint32_t output_y_begin = kernel_y == 0u ? 1u : 0u; + uint32_t output_y_end = kernel_y == 2u ? input_height / 2u : output_height; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 3u; ++kernel_x) { + uint32_t output_x_begin = kernel_x == 0u ? 1u : 0u; + uint32_t output_x_end = kernel_x == 2u && width != 0u ? width - 1u : width; + float weight = channel_weights[kernel_y * 3u + kernel_x]; + __m128 weight_values = _mm_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y * 2u + kernel_y - 1u; + const float* input_row = + input_channel + (size_t)((uint64_t)input_y * width); + float* output_row = output_channel + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 4u <= output_x_end; output_x += 4u) { + uint32_t input_x = output_x + kernel_x - 1u; + __m128 output_values = _mm_loadu_ps(output_row + output_x); + __m128 input_values = _mm_loadu_ps(input_row + input_x); + output_values = + _mm_add_ps(output_values, _mm_mul_ps(input_values, weight_values)); + _mm_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 1u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } +#else + lw_scalar_depthwise_conv3x3_stride2x1_pad1_f32( + input, weights, bias, output, input_dimensions, output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_depthwise5x5.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_depthwise5x5.c new file mode 100644 index 00000000..1e094882 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_depthwise5x5.c @@ -0,0 +1,75 @@ +#include "simd_kernels.h" +#include "simd_platform.h" + +#include "scalar_kernels.h" + +#include + +#define LW_COMPILES_SSE2_DEPTHWISE5X5 LW_SIMD_HAS_SSE2_INTRINSICS + +LW_SIMD_SSE2_TARGET +void lw_sse2_depthwise_conv5x5_unit_pad2_f32(const float* input, const float* weights, + const float* bias, float* output, + const int32_t dimensions[4]) { +#if LW_COMPILES_SSE2_DEPTHWISE5X5 + uint32_t height = (uint32_t)dimensions[2]; + uint32_t width = (uint32_t)dimensions[3]; + uint64_t channel_plane = (uint64_t)height * width; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)dimensions[0]; ++batch) { + uint32_t channel; + for (channel = 0u; channel < (uint32_t)dimensions[1]; ++channel) { + uint64_t channel_offset = + ((uint64_t)batch * (uint32_t)dimensions[1] + channel) * channel_plane; + const float* input_channel = input + (size_t)channel_offset; + const float* channel_weights = weights + (size_t)channel * 25u; + float* output_channel = output + (size_t)channel_offset; + float initial = bias == NULL ? 0.0f : bias[channel]; + __m128 initial_values = _mm_set1_ps(initial); + uint64_t spatial = 0u; + uint32_t kernel_y; + for (; spatial + 4u <= channel_plane; spatial += 4u) { + _mm_storeu_ps(output_channel + (size_t)spatial, initial_values); + } + for (; spatial < channel_plane; ++spatial) { + output_channel[(size_t)spatial] = initial; + } + for (kernel_y = 0u; kernel_y < 5u; ++kernel_y) { + uint32_t output_y_begin = kernel_y < 2u ? 2u - kernel_y : 0u; + uint32_t output_y_trim = kernel_y > 2u ? kernel_y - 2u : 0u; + uint32_t output_y_end = output_y_trim < height ? height - output_y_trim : 0u; + uint32_t kernel_x; + for (kernel_x = 0u; kernel_x < 5u; ++kernel_x) { + uint32_t output_x_begin = kernel_x < 2u ? 2u - kernel_x : 0u; + uint32_t output_x_trim = kernel_x > 2u ? kernel_x - 2u : 0u; + uint32_t output_x_end = output_x_trim < width ? width - output_x_trim : 0u; + float weight = channel_weights[kernel_y * 5u + kernel_x]; + __m128 weight_values = _mm_set1_ps(weight); + uint32_t output_y; + for (output_y = output_y_begin; output_y < output_y_end; ++output_y) { + uint32_t input_y = output_y + kernel_y - 2u; + const float* input_row = + input_channel + (size_t)((uint64_t)input_y * width); + float* output_row = output_channel + (size_t)((uint64_t)output_y * width); + uint32_t output_x = output_x_begin; + for (; output_x + 4u <= output_x_end; output_x += 4u) { + uint32_t input_x = output_x + kernel_x - 2u; + __m128 input_values = _mm_loadu_ps(input_row + input_x); + __m128 output_values = _mm_loadu_ps(output_row + output_x); + output_values = + _mm_add_ps(output_values, _mm_mul_ps(input_values, weight_values)); + _mm_storeu_ps(output_row + output_x, output_values); + } + for (; output_x < output_x_end; ++output_x) { + uint32_t input_x = output_x + kernel_x - 2u; + output_row[output_x] += input_row[input_x] * weight; + } + } + } + } + } + } +#else + lw_scalar_depthwise_conv5x5_unit_pad2_f32(input, weights, bias, output, dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_matmul.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_matmul.c new file mode 100644 index 00000000..f7bd699b --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_matmul.c @@ -0,0 +1,59 @@ +#include "simd_kernels.h" +#include "simd_platform.h" + +#include + +#define LW_COMPILES_SSE2 LW_SIMD_HAS_SSE2_INTRINSICS + +LW_SIMD_SSE2_TARGET +void lw_sse2_matmul_shared_f32( + const float* input, + const float* weights, + float* output, + uint32_t batch_count, + uint32_t rows, + uint32_t inner_dimension, + uint32_t columns) { + uint32_t batch; + uint32_t row; + for (batch = 0u; batch < batch_count; ++batch) { + for (row = 0u; row < rows;) { + uint32_t row_end = rows - row < 4u ? rows : row + 4u; + uint32_t inner; + uint32_t current_row; + for (current_row = row; current_row < row_end; ++current_row) { + uint64_t output_base = ((uint64_t)batch * rows + current_row) * columns; + uint32_t column; + for (column = 0u; column < columns; ++column) { + output[(size_t)(output_base + column)] = 0.0f; + } + } + for (inner = 0u; inner < inner_dimension; ++inner) { + uint64_t weight_base = (uint64_t)inner * columns; + for (current_row = row; current_row < row_end; ++current_row) { + uint64_t input_base = ((uint64_t)batch * rows + current_row) * inner_dimension; + uint64_t output_base = ((uint64_t)batch * rows + current_row) * columns; + float input_value = input[(size_t)(input_base + inner)]; + uint32_t column = 0u; +#if LW_COMPILES_SSE2 + __m128 input_values = _mm_set1_ps(input_value); + for (; column + 4u <= columns; column += 4u) { + __m128 output_values = + _mm_loadu_ps(output + (size_t)(output_base + column)); + __m128 weight_values = + _mm_loadu_ps(weights + (size_t)(weight_base + column)); + output_values = + _mm_add_ps(output_values, _mm_mul_ps(input_values, weight_values)); + _mm_storeu_ps(output + (size_t)(output_base + column), output_values); + } +#endif + for (; column < columns; ++column) { + output[(size_t)(output_base + column)] += + input_value * weights[(size_t)(weight_base + column)]; + } + } + } + row = row_end; + } + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_packed_conv1x1.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_packed_conv1x1.c new file mode 100644 index 00000000..7d781669 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/sse2_packed_conv1x1.c @@ -0,0 +1,119 @@ +#include "simd_kernels.h" +#include "simd_platform.h" + +#include "packed_conv_internal.h" + +#include + +#define LW_COMPILES_SSE2_PACKED_CONV1X1 LW_SIMD_HAS_SSE2_INTRINSICS + +LW_SIMD_SSE2_TARGET +void lw_sse2_packed_conv1x1_f32(const float* input, const float* packed_weights, + const float* bias, float* output, + const int32_t input_dimensions[4], + const int32_t output_dimensions[4]) { +#if LW_COMPILES_SSE2_PACKED_CONV1X1 + uint32_t input_channels = (uint32_t)input_dimensions[1]; + uint32_t output_channels = (uint32_t)output_dimensions[1]; + uint64_t channel_plane = + (uint64_t)(uint32_t)input_dimensions[2] * (uint32_t)input_dimensions[3]; + uint32_t output_blocks = + (output_channels + LW_PACKED_CONV1X1_OUTPUT_TILE - 1u) / LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t batch; + for (batch = 0u; batch < (uint32_t)input_dimensions[0]; ++batch) { + const float* batch_input = + input + (size_t)((uint64_t)batch * input_channels * channel_plane); + float* batch_output = output + (size_t)((uint64_t)batch * output_channels * channel_plane); + uint32_t output_block; + for (output_block = 0u; output_block < output_blocks; ++output_block) { + uint32_t output_base = output_block * LW_PACKED_CONV1X1_OUTPUT_TILE; + uint32_t valid_outputs = output_channels - output_base; + uint64_t spatial = 0u; + if (valid_outputs > LW_PACKED_CONV1X1_OUTPUT_TILE) { + valid_outputs = LW_PACKED_CONV1X1_OUTPUT_TILE; + } + for (; spatial + 4u <= channel_plane; spatial += 4u) { + __m128 accumulator0 = _mm_set1_ps(bias == NULL ? 0.0f : bias[output_base]); + __m128 accumulator1 = _mm_set1_ps( + bias == NULL || valid_outputs <= 1u ? 0.0f : bias[output_base + 1u]); + __m128 accumulator2 = _mm_set1_ps( + bias == NULL || valid_outputs <= 2u ? 0.0f : bias[output_base + 2u]); + __m128 accumulator3 = _mm_set1_ps( + bias == NULL || valid_outputs <= 3u ? 0.0f : bias[output_base + 3u]); + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + const float* input_channel_data = + batch_input + (size_t)((uint64_t)input_channel * channel_plane + spatial); + const float* packed = + packed_weights + + (size_t)(((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE); + __m128 input_values = _mm_loadu_ps(input_channel_data); + accumulator0 = + _mm_add_ps(accumulator0, _mm_mul_ps(input_values, _mm_set1_ps(packed[0]))); + accumulator1 = + _mm_add_ps(accumulator1, _mm_mul_ps(input_values, _mm_set1_ps(packed[1]))); + accumulator2 = + _mm_add_ps(accumulator2, _mm_mul_ps(input_values, _mm_set1_ps(packed[2]))); + accumulator3 = + _mm_add_ps(accumulator3, _mm_mul_ps(input_values, _mm_set1_ps(packed[3]))); + } + _mm_storeu_ps(batch_output + + (size_t)((uint64_t)output_base * channel_plane + spatial), + accumulator0); + if (valid_outputs > 1u) + _mm_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 1u) * channel_plane + spatial), + accumulator1); + if (valid_outputs > 2u) + _mm_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 2u) * channel_plane + spatial), + accumulator2); + if (valid_outputs > 3u) + _mm_storeu_ps( + batch_output + + (size_t)((uint64_t)(output_base + 3u) * channel_plane + spatial), + accumulator3); + } + for (; spatial < channel_plane; ++spatial) { + float accumulator0 = bias == NULL ? 0.0f : bias[output_base]; + float accumulator1 = + bias == NULL || valid_outputs <= 1u ? 0.0f : bias[output_base + 1u]; + float accumulator2 = + bias == NULL || valid_outputs <= 2u ? 0.0f : bias[output_base + 2u]; + float accumulator3 = + bias == NULL || valid_outputs <= 3u ? 0.0f : bias[output_base + 3u]; + uint32_t input_channel; + for (input_channel = 0u; input_channel < input_channels; ++input_channel) { + float input_value = + batch_input[(size_t)((uint64_t)input_channel * channel_plane + spatial)]; + const float* packed = + packed_weights + + (size_t)(((uint64_t)output_block * input_channels + input_channel) * + LW_PACKED_CONV1X1_OUTPUT_TILE); + accumulator0 += input_value * packed[0]; + accumulator1 += input_value * packed[1]; + accumulator2 += input_value * packed[2]; + accumulator3 += input_value * packed[3]; + } + batch_output[(size_t)((uint64_t)output_base * channel_plane + spatial)] = + accumulator0; + if (valid_outputs > 1u) + batch_output[(size_t)((uint64_t)(output_base + 1u) * channel_plane + spatial)] = + accumulator1; + if (valid_outputs > 2u) + batch_output[(size_t)((uint64_t)(output_base + 2u) * channel_plane + spatial)] = + accumulator2; + if (valid_outputs > 3u) + batch_output[(size_t)((uint64_t)(output_base + 3u) * channel_plane + spatial)] = + accumulator3; + } + } + } +#else + lw_scalar_packed_conv1x1_f32(input, packed_weights, bias, output, input_dimensions, + output_dimensions); +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_bitmap.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_bitmap.c new file mode 100644 index 00000000..c3b4ff76 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_bitmap.c @@ -0,0 +1,35 @@ +#include "simd_kernels.h" + +#include +#include +#include +#include + +/* Preserve the AVX2/scalar DB contract: reject NaN/Inf and set each byte only + * when probability is strictly greater than the threshold. */ +int lw_wasm128_threshold_bitmap_f32(const float* prediction, uint8_t* bitmap, + uint64_t pixel_count, float threshold) { + const v128_t absolute_mask = wasm_i32x4_splat(0x7fffffff); + const v128_t infinity_bits = wasm_i32x4_splat(0x7f800000); + const v128_t limit = wasm_f32x4_splat(threshold); + uint64_t index = 0u; + if (prediction == NULL || bitmap == NULL) return -1; + for (; index + 4u <= pixel_count; index += 4u) { + v128_t values = wasm_v128_load(prediction + (size_t)index); + v128_t absolute_bits = wasm_v128_and(values, absolute_mask); + uint32_t mask; + if (wasm_i32x4_bitmask(wasm_i32x4_ge(absolute_bits, infinity_bits)) != 0) + return -1; + mask = (uint32_t)wasm_i32x4_bitmask(wasm_f32x4_gt(values, limit)); + bitmap[(size_t)index] = (uint8_t)(mask & 1u); + bitmap[(size_t)index + 1u] = (uint8_t)((mask >> 1u) & 1u); + bitmap[(size_t)index + 2u] = (uint8_t)((mask >> 2u) & 1u); + bitmap[(size_t)index + 3u] = (uint8_t)((mask >> 3u) & 1u); + } + for (; index < pixel_count; ++index) { + float value = prediction[(size_t)index]; + if (!isfinite(value)) return -1; + bitmap[(size_t)index] = value > threshold ? 1u : 0u; + } + return 0; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_elementwise.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_elementwise.c new file mode 100644 index 00000000..b76a0e34 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_elementwise.c @@ -0,0 +1,226 @@ +#include "simd_kernels.h" + +#include +#include +#include +#include +#include + +static v128_t binary_vector(lw_scalar_binary_op operation, v128_t left, v128_t right) { + switch (operation) { + case LW_SCALAR_BINARY_ADD: return wasm_f32x4_add(left, right); + case LW_SCALAR_BINARY_SUB: return wasm_f32x4_sub(left, right); + case LW_SCALAR_BINARY_MUL: return wasm_f32x4_mul(left, right); + case LW_SCALAR_BINARY_DIV: return wasm_f32x4_div(left, right); + default: return left; + } +} + +static float binary_scalar(lw_scalar_binary_op operation, float left, float right) { + switch (operation) { + case LW_SCALAR_BINARY_ADD: return left + right; + case LW_SCALAR_BINARY_SUB: return left - right; + case LW_SCALAR_BINARY_MUL: return left * right; + case LW_SCALAR_BINARY_DIV: return left / right; + default: return powf(left, right); + } +} + +void lw_wasm128_binary_contiguous_f32(lw_scalar_binary_op operation, + const float* left, const float* right, float* output, uint64_t count) { + uint64_t index = 0u; + if (operation != LW_SCALAR_BINARY_POW) { + for (; index + 4u <= count; index += 4u) { + wasm_v128_store(output + (size_t)index, + binary_vector(operation, wasm_v128_load(left + (size_t)index), + wasm_v128_load(right + (size_t)index))); + } + } + for (; index < count; ++index) + output[(size_t)index] = binary_scalar(operation, left[(size_t)index], right[(size_t)index]); +} + +void lw_wasm128_binary_scalar_f32(lw_scalar_binary_op operation, + const float* tensor, float scalar, float* output, uint64_t count, int scalar_is_left) { + uint64_t index = 0u; + if (operation != LW_SCALAR_BINARY_POW) { + v128_t broadcast = wasm_f32x4_splat(scalar); + for (; index + 4u <= count; index += 4u) { + v128_t value = wasm_v128_load(tensor + (size_t)index); + wasm_v128_store(output + (size_t)index, + binary_vector(operation, scalar_is_left != 0 ? broadcast : value, + scalar_is_left != 0 ? value : broadcast)); + } + } + for (; index < count; ++index) { + float value = tensor[(size_t)index]; + output[(size_t)index] = binary_scalar(operation, + scalar_is_left != 0 ? scalar : value, + scalar_is_left != 0 ? value : scalar); + } +} + +void lw_wasm128_binary_channel_nhwc_f32(lw_scalar_binary_op operation, + const float* tensor, const float* channel, float* output, + uint64_t pixels, uint32_t channels, int channel_is_left) { + for (uint64_t pixel = 0u; pixel < pixels; ++pixel) { + const float* source = tensor + (size_t)pixel * channels; + float* destination = output + (size_t)pixel * channels; + uint32_t c = 0u; + if (operation != LW_SCALAR_BINARY_POW) { + for (; c + 4u <= channels; c += 4u) { + v128_t value = wasm_v128_load(source + c); + v128_t scale = wasm_v128_load(channel + c); + wasm_v128_store(destination + c, + binary_vector(operation, channel_is_left != 0 ? scale : value, + channel_is_left != 0 ? value : scale)); + } + } + for (; c < channels; ++c) { + destination[c] = binary_scalar(operation, + channel_is_left != 0 ? channel[c] : source[c], + channel_is_left != 0 ? source[c] : channel[c]); + } + } +} + +void lw_wasm128_affine_nhwc_f32(const float* input, const float* mul, + const float* add, float* output, uint32_t pixels, uint32_t channels) { + for (uint32_t pixel = 0u; pixel < pixels; ++pixel) { + const float* source = input + (size_t)pixel * channels; + float* destination = output + (size_t)pixel * channels; + uint32_t channel = 0u; + for (; channel + 4u <= channels; channel += 4u) { + v128_t value = wasm_f32x4_mul(wasm_v128_load(source + channel), + wasm_v128_load(mul + channel)); + wasm_v128_store(destination + channel, + wasm_f32x4_add(value, wasm_v128_load(add + channel))); + } + for (; channel < channels; ++channel) + destination[channel] = source[channel] * mul[channel] + add[channel]; + } +} + +void lw_wasm128_relu_f32(const float* input, float* output, uint64_t count) { + const v128_t zero = wasm_f32x4_splat(0.0f); + uint64_t index = 0u; + for (; index + 4u <= count; index += 4u) { + v128_t value = wasm_v128_load(input + (size_t)index); + wasm_v128_store(output + (size_t)index, + wasm_v128_bitselect(value, zero, wasm_f32x4_gt(value, zero))); + } + for (; index < count; ++index) { + float value = input[(size_t)index]; + output[(size_t)index] = value > 0.0f ? value : 0.0f; + } +} + +void lw_wasm128_reduce_mean_hw_f32(const float* input, float* output, + uint32_t batch, uint32_t height, uint32_t width, uint32_t channels) { + const float denominator = (float)height * (float)width; + for (uint32_t n = 0u; n < batch; ++n) { + uint32_t channel = 0u; + for (; channel + 4u <= channels; channel += 4u) { + v128_t sum = wasm_f32x4_splat(0.0f); + for (uint32_t y = 0u; y < height; ++y) { + for (uint32_t x = 0u; x < width; ++x) { + const float* source = input + + ((((size_t)n * height + y) * width + x) * channels + channel); + sum = wasm_f32x4_add(sum, wasm_v128_load(source)); + } + } + wasm_v128_store(output + (size_t)n * channels + channel, + wasm_f32x4_div(sum, wasm_f32x4_splat(denominator))); + } + for (; channel < channels; ++channel) { + float sum = 0.0f; + for (uint32_t y = 0u; y < height; ++y) + for (uint32_t x = 0u; x < width; ++x) + sum += input[(((size_t)n * height + y) * width + x) * channels + channel]; + output[(size_t)n * channels + channel] = sum / denominator; + } + } +} + +static void pool_window(int32_t iy0, int32_t ix0, uint32_t input_height, + uint32_t input_width, uint32_t kernel_h, uint32_t kernel_w, + uint32_t* ky_begin, uint32_t* ky_end, uint32_t* kx_begin, uint32_t* kx_end) { + int32_t y_begin = iy0 < 0 ? -iy0 : 0; + int32_t x_begin = ix0 < 0 ? -ix0 : 0; + int32_t y_end = (int32_t)kernel_h; + int32_t x_end = (int32_t)kernel_w; + if (iy0 + y_end > (int32_t)input_height) y_end = (int32_t)input_height - iy0; + if (ix0 + x_end > (int32_t)input_width) x_end = (int32_t)input_width - ix0; + if (y_end < y_begin) y_end = y_begin; + if (x_end < x_begin) x_end = x_begin; + if (y_begin > (int32_t)kernel_h) y_begin = (int32_t)kernel_h; + if (x_begin > (int32_t)kernel_w) x_begin = (int32_t)kernel_w; + if (y_end > (int32_t)kernel_h) y_end = (int32_t)kernel_h; + if (x_end > (int32_t)kernel_w) x_end = (int32_t)kernel_w; + *ky_begin = (uint32_t)y_begin; + *ky_end = (uint32_t)y_end; + *kx_begin = (uint32_t)x_begin; + *kx_end = (uint32_t)x_end; +} + +void lw_wasm128_pool_nhwc_f32(const float* input, float* output, + uint32_t batch, uint32_t input_height, uint32_t input_width, + uint32_t output_height, uint32_t output_width, uint32_t channels, + uint32_t kernel_h, uint32_t kernel_w, uint32_t stride_h, uint32_t stride_w, + uint32_t pad_top, uint32_t pad_left, uint8_t count_include_pad, uint8_t is_max) { + for (uint32_t n = 0u; n < batch; ++n) { + for (uint32_t oy = 0u; oy < output_height; ++oy) { + for (uint32_t ox = 0u; ox < output_width; ++ox) { + int32_t iy0 = (int32_t)((uint64_t)oy * stride_h) - (int32_t)pad_top; + int32_t ix0 = (int32_t)((uint64_t)ox * stride_w) - (int32_t)pad_left; + uint32_t ky_begin, ky_end, kx_begin, kx_end; + uint32_t channel = 0u; + uint32_t denominator; + float* destination = output + + ((((size_t)n * output_height + oy) * output_width + ox) * channels); + pool_window(iy0, ix0, input_height, input_width, kernel_h, kernel_w, + &ky_begin, &ky_end, &kx_begin, &kx_end); + denominator = count_include_pad ? kernel_h * kernel_w + : (ky_end - ky_begin) * (kx_end - kx_begin); + if (denominator == 0u) denominator = 1u; + for (; channel + 4u <= channels; channel += 4u) { + v128_t value = wasm_f32x4_splat(is_max ? -FLT_MAX : 0.0f); + for (uint32_t ky = ky_begin; ky < ky_end; ++ky) { + uint32_t iy = (uint32_t)(iy0 + (int32_t)ky); + for (uint32_t kx = kx_begin; kx < kx_end; ++kx) { + uint32_t ix = (uint32_t)(ix0 + (int32_t)kx); + const float* source = input + + ((((size_t)n * input_height + iy) * input_width + ix) * + channels + channel); + v128_t sample = wasm_v128_load(source); + value = is_max + ? wasm_v128_bitselect(sample, value, + wasm_f32x4_gt(sample, value)) + : wasm_f32x4_add(value, sample); + } + } + if (!is_max) + value = wasm_f32x4_div(value, + wasm_f32x4_splat((float)denominator)); + wasm_v128_store(destination + channel, value); + } + for (; channel < channels; ++channel) { + float value = is_max ? -FLT_MAX : 0.0f; + for (uint32_t ky = ky_begin; ky < ky_end; ++ky) { + uint32_t iy = (uint32_t)(iy0 + (int32_t)ky); + for (uint32_t kx = kx_begin; kx < kx_end; ++kx) { + uint32_t ix = (uint32_t)(ix0 + (int32_t)kx); + float sample = input[(((size_t)n * input_height + iy) * + input_width + ix) * channels + channel]; + if (is_max) { + if (sample > value) value = sample; + } else value += sample; + } + } + if (!is_max) value /= (float)denominator; + destination[channel] = value; + } + } + } + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_epilogue_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_epilogue_internal.h new file mode 100644 index 00000000..923dc2f2 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_epilogue_internal.h @@ -0,0 +1,59 @@ +#ifndef LW_WASM128_EPILOGUE_INTERNAL_H +#define LW_WASM128_EPILOGUE_INTERNAL_H + +#include "nhwc_internal.h" +#include "simd_kernels.h" +#include + +#if defined(__EMSCRIPTEN__) && defined(__wasm_simd128__) +#include +#include "wasm128_erf_internal.h" + +/* Full OC16 blocks only: callers handle the channel tail without reading + * past post_bias or residual. Bias is already in the accumulators. */ +static inline void lw_wasm128_apply_epilogue_oc16( + v128_t values[4], const lw_nhwc_epilogue* epilogue, + uint32_t channel_base, const float* residual, int hardswish_divide) { + uint32_t group; + if (epilogue == NULL) return; + if (epilogue->post_bias != NULL) { + for (group = 0u; group < 4u; ++group) { + values[group] = wasm_f32x4_add(values[group], + wasm_v128_load(epilogue->post_bias + channel_base + group * 4u)); + } + } + if (residual != NULL) { + for (group = 0u; group < 4u; ++group) { + values[group] = wasm_f32x4_add(values[group], + wasm_v128_load(residual + group * 4u)); + } + } + if (epilogue->activation == LW_NHWC_ACT_RELU) { + const v128_t zero = wasm_f32x4_splat(0.0f); + for (group = 0u; group < 4u; ++group) { + values[group] = wasm_v128_bitselect(values[group], zero, + wasm_f32x4_gt(values[group], zero)); + } + } else if (epilogue->activation == LW_NHWC_ACT_HARDSWISH) { + const v128_t zero = wasm_f32x4_splat(0.0f); + const v128_t three = wasm_f32x4_splat(3.0f); + const v128_t six = wasm_f32x4_splat(6.0f); + const v128_t inverse_six = wasm_f32x4_splat(1.0f / 6.0f); + for (group = 0u; group < 4u; ++group) { + v128_t gate = wasm_f32x4_add(values[group], three); + gate = wasm_v128_bitselect(zero, gate, wasm_f32x4_lt(gate, zero)); + gate = wasm_v128_bitselect(six, gate, wasm_f32x4_gt(gate, six)); + gate = wasm_f32x4_mul(values[group], gate); + values[group] = hardswish_divide != 0 + ? wasm_f32x4_div(gate, six) + : wasm_f32x4_mul(gate, inverse_six); + } + } else if (epilogue->activation == LW_NHWC_ACT_GELU) { + for (group = 0u; group < 4u; ++group) { + values[group] = lw_wasm128_gelu_vector_f32(values[group]); + } + } +} +#endif + +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_erf.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_erf.c new file mode 100644 index 00000000..d960bd7f --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_erf.c @@ -0,0 +1,63 @@ +#include "simd_kernels.h" + +/* WASM SIMD128 Erf approximation shared by browser, Node, and compiled REC. */ + +#include +#include +#include + +#if defined(__EMSCRIPTEN__) && defined(__wasm_simd128__) +# include +# define LW_COMPILES_WASM128_ERF 1 +#else +# define LW_COMPILES_WASM128_ERF 0 +#endif + +#if LW_COMPILES_WASM128_ERF +#include "wasm128_erf_internal.h" +#endif + +void lw_wasm128_erf_f32(const float* input, float* output, uint64_t element_count) { +#if LW_COMPILES_WASM128_ERF + uint64_t index = 0u; + + for (; index + 4u <= element_count; index += 4u) { + wasm_v128_store(output + (size_t)index, + erf_approximation_f32(wasm_v128_load(input + (size_t)index))); + } + for (; index < element_count; ++index) { + output[(size_t)index] = erff(input[(size_t)index]); + } +#else + uint64_t index; + for (index = 0u; index < element_count; ++index) { + output[(size_t)index] = erff(input[(size_t)index]); + } +#endif +} + +/* Reuse the canonical WASM Erf polynomial in fused REC GELU epilogues. + * Input and output may alias. */ +void lw_wasm128_gelu_f32(const float* input, float* output, uint64_t element_count) { +#if LW_COMPILES_WASM128_ERF + uint64_t index = 0u; + for (; index + 4u <= element_count; index += 4u) { + v128_t value = wasm_v128_load(input + (size_t)index); + wasm_v128_store(output + (size_t)index, lw_wasm128_gelu_vector_f32(value)); + } + for (; index < element_count; ++index) { + float value = input[(size_t)index]; + float activated = erff(value / 1.4142135381698608f) + 1.0f; + activated = value * activated; + output[(size_t)index] = activated * 0.5f; + } +#else + uint64_t index; + for (index = 0u; index < element_count; ++index) { + float value = input[(size_t)index]; + float activated = erff(value / 1.4142135381698608f) + 1.0f; + activated = value * activated; + output[(size_t)index] = activated * 0.5f; + } +#endif +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_erf_internal.h b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_erf_internal.h new file mode 100644 index 00000000..9e9b25c2 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_erf_internal.h @@ -0,0 +1,103 @@ +#ifndef LW_WASM128_ERF_INTERNAL_H +#define LW_WASM128_ERF_INTERNAL_H + +#include +#if defined(__EMSCRIPTEN__) && defined(__wasm_simd128__) +#include + +static inline v128_t erf_small_magnitude_f32(v128_t absolute, v128_t squared) { + v128_t polynomial = wasm_f32x4_splat(1.0590875083315439e-6f); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-1.3906452410711274e-5f), + wasm_f32x4_mul(squared, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(1.1955437252428243e-4f), + wasm_f32x4_mul(squared, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-8.5424759600797662e-4f), + wasm_f32x4_mul(squared, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(5.2237718994271529e-3f), + wasm_f32x4_mul(squared, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-2.6866128888288668e-2f), + wasm_f32x4_mul(squared, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(1.1283791226625459e-1f), + wasm_f32x4_mul(squared, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-3.7612638883043881e-1f), + wasm_f32x4_mul(squared, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(1.1283791670929921f), + wasm_f32x4_mul(squared, polynomial)); + return wasm_f32x4_mul(absolute, polynomial); +} + +static inline v128_t erf_middle_magnitude_f32(v128_t absolute) { + v128_t shifted = wasm_f32x4_sub(absolute, wasm_f32x4_splat(1.5f)); + v128_t polynomial = wasm_f32x4_splat(-2.4006675278365739e-3f); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-3.8855162788028288e-3f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(1.9332860298601401e-2f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-1.5013607234871428e-2f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-4.4599369472787913e-2f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(1.3876136033191724e-1f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-1.7839541988688759e-1f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(1.1893013063163335e-1f), + wasm_f32x4_mul(shifted, polynomial)); + return wasm_f32x4_add(wasm_f32x4_splat(9.6610514641406819e-1f), + wasm_f32x4_mul(shifted, polynomial)); +} + +static inline v128_t erf_large_magnitude_f32(v128_t absolute) { + v128_t shifted = wasm_f32x4_sub(absolute, wasm_f32x4_splat(3.0f)); + v128_t polynomial = wasm_f32x4_splat(-8.8750765320565021e-5f); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(3.8805285630072217e-4f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-7.7812010711428433e-4f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(1.0255980996254569e-3f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-1.0307062838910627e-3f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(7.8586080129729565e-4f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(-4.1936053855221229e-4f), + wasm_f32x4_mul(shifted, polynomial)); + polynomial = wasm_f32x4_add(wasm_f32x4_splat(1.3951109720745927e-4f), + wasm_f32x4_mul(shifted, polynomial)); + return wasm_f32x4_add(wasm_f32x4_splat(9.9997793886838715e-1f), + wasm_f32x4_mul(shifted, polynomial)); +} + +static inline v128_t erf_approximation_f32(v128_t value) { + const v128_t absolute_mask = wasm_i32x4_splat(INT32_C(0x7fffffff)); + const v128_t sign_mask = wasm_i32x4_splat(INT32_MIN); + const v128_t one = wasm_f32x4_splat(1.0f); + v128_t absolute = wasm_v128_and(value, absolute_mask); + v128_t squared = wasm_f32x4_mul(absolute, absolute); + /* When all lanes are in the small-magnitude interval, avoid evaluating + * the other two polynomials. Mixed blocks and + * NaNs retain the original lane-select path below, bit for bit. */ + if (wasm_i32x4_all_true(wasm_f32x4_lt(absolute, one))) { + v128_t small = erf_small_magnitude_f32(absolute, squared); + return wasm_v128_xor(small, wasm_v128_and(value, sign_mask)); + } + v128_t result = wasm_v128_bitselect(erf_middle_magnitude_f32(absolute), + erf_large_magnitude_f32(absolute), + wasm_f32x4_lt(absolute, wasm_f32x4_splat(2.0f))); + result = wasm_v128_bitselect(erf_small_magnitude_f32(absolute, squared), + result, wasm_f32x4_lt(absolute, one)); + result = wasm_v128_bitselect(one, result, + wasm_f32x4_ge(absolute, wasm_f32x4_splat(4.0f))); + result = wasm_v128_xor(result, wasm_v128_and(value, sign_mask)); + return wasm_v128_bitselect(value, result, wasm_f32x4_ne(value, value)); +} + +/* Keep the canonical divide/add/mul/mul sequence; no reciprocal or FMA. */ +static inline v128_t lw_wasm128_gelu_vector_f32(v128_t value) { + v128_t scaled = wasm_f32x4_div(value, wasm_f32x4_splat(1.4142135381698608f)); + v128_t activated = wasm_f32x4_add(erf_approximation_f32(scaled), wasm_f32x4_splat(1.0f)); + activated = wasm_f32x4_mul(value, activated); + return wasm_f32x4_mul(activated, wasm_f32x4_splat(0.5f)); +} +#endif +#endif diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_matmul.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_matmul.c new file mode 100644 index 00000000..c6fc6e81 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_matmul.c @@ -0,0 +1,61 @@ +#include "simd_kernels.h" +#include "../kernels/packed_matmul_internal.h" + +#include +#include +#include + +/* Shared packed MatMul [panel][inner][OC16], four rows at a time. Keep the + * inner loop order unchanged so every output lane has the same accumulation + * sequence as the scalar physical backend. */ +void lw_wasm128_packed_matmul_shared_f32(const float* input, + const float* packed_weights, float* output, uint32_t batch_count, + uint32_t rows, uint32_t inner_dimension, uint32_t columns) { + for (uint32_t batch = 0u; batch < batch_count; ++batch) { + for (uint32_t column_base = 0u; column_base < columns; + column_base += LW_PACKED_MATMUL_COLUMN_TILE) { + const float* panel = packed_weights + + (size_t)(column_base / LW_PACKED_MATMUL_COLUMN_TILE) * + inner_dimension * LW_PACKED_MATMUL_COLUMN_TILE; + uint32_t valid_columns = columns - column_base; + if (valid_columns > LW_PACKED_MATMUL_COLUMN_TILE) + valid_columns = LW_PACKED_MATMUL_COLUMN_TILE; + for (uint32_t row_base = 0u; row_base < rows; row_base += 4u) { + v128_t sums[4][4]; + uint32_t row_count = rows - row_base; + if (row_count > 4u) row_count = 4u; + for (uint32_t row = 0u; row < row_count; ++row) + for (uint32_t group = 0u; group < 4u; ++group) + sums[row][group] = wasm_f32x4_splat(0.0f); + for (uint32_t inner = 0u; inner < inner_dimension; ++inner) { + const float* packed = panel + + (size_t)inner * LW_PACKED_MATMUL_COLUMN_TILE; + v128_t weights[4]; + for (uint32_t group = 0u; group < 4u; ++group) + weights[group] = wasm_v128_load(packed + group * 4u); + for (uint32_t row = 0u; row < row_count; ++row) { + v128_t value = wasm_f32x4_splat(input[ + ((size_t)batch * rows + row_base + row) * inner_dimension + inner]); + for (uint32_t group = 0u; group < 4u; ++group) + sums[row][group] = wasm_f32x4_add(sums[row][group], + wasm_f32x4_mul(value, weights[group])); + } + } + for (uint32_t row = 0u; row < row_count; ++row) { + float* destination = output + + ((size_t)batch * rows + row_base + row) * columns + column_base; + if (valid_columns == LW_PACKED_MATMUL_COLUMN_TILE) { + for (uint32_t group = 0u; group < 4u; ++group) + wasm_v128_store(destination + group * 4u, sums[row][group]); + } else { + float tail[LW_PACKED_MATMUL_COLUMN_TILE]; + for (uint32_t group = 0u; group < 4u; ++group) + wasm_v128_store(tail + group * 4u, sums[row][group]); + for (uint32_t lane = 0u; lane < valid_columns; ++lane) + destination[lane] = tail[lane]; + } + } + } + } + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_nhwc_convtranspose.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_nhwc_convtranspose.c new file mode 100644 index 00000000..d02cb476 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_nhwc_convtranspose.c @@ -0,0 +1,119 @@ +#include "../kernels/nhwc_internal.h" + +#include +#include + +static int valid_shape(const lw_nhwc_convtranspose_desc* desc) { + return desc != NULL && desc->batch != 0u && desc->input_channels != 0u && + desc->input_height != 0u && desc->input_width != 0u && + desc->output_channels != 0u && + desc->input_height <= UINT32_MAX / 2u && + desc->input_width <= UINT32_MAX / 2u && + desc->output_height == desc->input_height * 2u && + desc->output_width == desc->input_width * 2u; +} + +static v128_t relu_vector(v128_t value) { + v128_t zero = wasm_f32x4_splat(0.0f); + return wasm_v128_bitselect(value, zero, wasm_f32x4_gt(value, zero)); +} + +lw_status lw_wasm128_nhwc_convtranspose2x2_s2_f32( + const float* input, const float* packed_weights, const lw_nhwc_epilogue* epilogue, + float* output, const lw_nhwc_convtranspose_desc* desc) { + uint32_t blocks; + if (input == NULL || packed_weights == NULL || output == NULL || + !valid_shape(desc) || (desc->output_channels & 7u) != 0u) { + return LW_STATUS_INVALID_ARGUMENT; + } + blocks = (desc->output_channels + LW_NHWC_OC_BLOCK - 1u) / LW_NHWC_OC_BLOCK; + for (uint32_t n = 0u; n < desc->batch; ++n) { + for (uint32_t iy = 0u; iy < desc->input_height; ++iy) { + for (uint32_t ix = 0u; ix < desc->input_width; ++ix) { + const float* source = input + + (((size_t)n * desc->input_height + iy) * desc->input_width + ix) * + desc->input_channels; + for (uint32_t tap = 0u; tap < 4u; ++tap) { + uint32_t oy = iy * 2u + (tap >> 1u); + uint32_t ox = ix * 2u + (tap & 1u); + float* destination = output + + (((size_t)n * desc->output_height + oy) * desc->output_width + ox) * + desc->output_channels; + for (uint32_t block = 0u; block < blocks; ++block) { + uint32_t base = block * LW_NHWC_OC_BLOCK; + uint32_t valid = desc->output_channels - base; + v128_t acc[4]; + float temporary[LW_NHWC_OC_BLOCK]; + if (valid > LW_NHWC_OC_BLOCK) valid = LW_NHWC_OC_BLOCK; + for (uint32_t lane = 0u; lane < LW_NHWC_OC_BLOCK; ++lane) + temporary[lane] = epilogue != NULL && epilogue->bias != NULL && + lane < valid ? epilogue->bias[base + lane] : 0.0f; + for (uint32_t vector = 0u; vector < 4u; ++vector) + acc[vector] = wasm_v128_load(temporary + vector * 4u); + for (uint32_t ic = 0u; ic < desc->input_channels; ++ic) { + const float* weight = packed_weights + + (((size_t)tap * blocks + block) * desc->input_channels + ic) * + LW_NHWC_OC_BLOCK; + v128_t sample = wasm_f32x4_splat(source[ic]); + for (uint32_t vector = 0u; vector < 4u; ++vector) { + acc[vector] = wasm_f32x4_add(acc[vector], + wasm_f32x4_mul(sample, + wasm_v128_load(weight + vector * 4u))); + } + } + for (uint32_t vector = 0u; vector < 4u; ++vector) { + if (epilogue != NULL && + epilogue->activation == LW_NHWC_ACT_RELU) + acc[vector] = relu_vector(acc[vector]); + } + if (valid == LW_NHWC_OC_BLOCK) { + for (uint32_t vector = 0u; vector < 4u; ++vector) + wasm_v128_store(destination + base + vector * 4u, acc[vector]); + } else { + for (uint32_t vector = 0u; vector < 4u; ++vector) + wasm_v128_store(temporary + vector * 4u, acc[vector]); + for (uint32_t lane = 0u; lane < valid; ++lane) + destination[base + lane] = temporary[lane]; + } + } + } + } + } + } + return LW_STATUS_OK; +} + +lw_status lw_wasm128_nhwc_convtranspose2x2_s2_c1_f32( + const float* input, const float* weights, const lw_nhwc_epilogue* epilogue, + float* output, const lw_nhwc_convtranspose_desc* desc) { + if (input == NULL || weights == NULL || output == NULL || + !valid_shape(desc) || desc->output_channels != 1u) { + return LW_STATUS_INVALID_ARGUMENT; + } + for (uint32_t n = 0u; n < desc->batch; ++n) { + for (uint32_t iy = 0u; iy < desc->input_height; ++iy) { + for (uint32_t ix = 0u; ix < desc->input_width; ++ix) { + const float* source = input + + (((size_t)n * desc->input_height + iy) * desc->input_width + ix) * + desc->input_channels; + v128_t acc = wasm_f32x4_splat( + epilogue != NULL && epilogue->bias != NULL ? epilogue->bias[0] : 0.0f); + float taps[4]; + for (uint32_t ic = 0u; ic < desc->input_channels; ++ic) { + acc = wasm_f32x4_add(acc, wasm_f32x4_mul( + wasm_f32x4_splat(source[ic]), wasm_v128_load(weights + (size_t)ic * 4u))); + } + if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_RELU) + acc = relu_vector(acc); + wasm_v128_store(taps, acc); + for (uint32_t tap = 0u; tap < 4u; ++tap) { + uint32_t oy = iy * 2u + (tap >> 1u); + uint32_t ox = ix * 2u + (tap & 1u); + output[((size_t)n * desc->output_height + oy) * + desc->output_width + ox] = taps[tap]; + } + } + } + } + return LW_STATUS_OK; +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_nhwc_pointwise.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_nhwc_pointwise.c new file mode 100644 index 00000000..afbbdb24 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_nhwc_pointwise.c @@ -0,0 +1,256 @@ +#include "nhwc_internal.h" +#include "simd_kernels.h" +#include "wasm128_epilogue_internal.h" + +#include +#include +#include + +static float apply_activation(float value, uint16_t activation) { + if (activation == LW_NHWC_ACT_RELU) return value > 0.0f ? value : 0.0f; + if (activation == LW_NHWC_ACT_HARDSWISH) { + float gate = value + 3.0f; + if (gate < 0.0f) gate = 0.0f; + else if (gate > 6.0f) gate = 6.0f; + return value * gate / 6.0f; + } + if (activation == LW_NHWC_ACT_GELU) { + return lw_nhwc_gelu_scalar_exact_f32(value); + } + return value; +} + +/* Keep the shared OC16 pack format. One weight load serves up to four + * independent pixels; standard SIMD128 mul/add preserves non-FMA semantics. */ +void lw_wasm128_nhwc_pointwise_4x16_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels) { + for (uint32_t pixel = 0u; pixel < pixels; pixel += 4u) { + uint32_t rows = pixels - pixel; + if (rows > 4u) rows = 4u; + for (uint32_t oc = 0u; oc < output_channels; oc += LW_NHWC_OC_BLOCK) { + uint32_t lanes = output_channels - oc; + float bias_values[LW_NHWC_OC_BLOCK] = {0.0f}; + v128_t initial[4]; + v128_t accumulators[4][4]; + const float* weights = packed_weights + + (size_t)(oc / LW_NHWC_OC_BLOCK) * input_channels * LW_NHWC_OC_BLOCK; + if (lanes > LW_NHWC_OC_BLOCK) lanes = LW_NHWC_OC_BLOCK; + if (epilogue != NULL && epilogue->bias != NULL) { + for (uint32_t lane = 0u; lane < lanes; ++lane) { + bias_values[lane] = epilogue->bias[oc + lane]; + } + } + for (uint32_t group = 0u; group < 4u; ++group) { + initial[group] = wasm_v128_load(bias_values + group * 4u); + } + for (uint32_t row = 0u; row < rows; ++row) { + for (uint32_t group = 0u; group < 4u; ++group) { + accumulators[row][group] = initial[group]; + } + } + for (uint32_t ic = 0u; ic < input_channels; ++ic) { + const float* packed = weights + (size_t)ic * LW_NHWC_OC_BLOCK; + v128_t weight[4]; + for (uint32_t group = 0u; group < 4u; ++group) { + weight[group] = wasm_v128_load(packed + group * 4u); + } + for (uint32_t row = 0u; row < rows; ++row) { + v128_t value = wasm_f32x4_splat( + input[(size_t)(pixel + row) * input_channels + ic]); + for (uint32_t group = 0u; group < 4u; ++group) { + accumulators[row][group] = wasm_f32x4_add( + accumulators[row][group], + wasm_f32x4_mul(value, weight[group])); + } + } + } + for (uint32_t row = 0u; row < rows; ++row) { + size_t output_base = (size_t)(pixel + row) * output_channels + oc; + if (lanes == LW_NHWC_OC_BLOCK) { + v128_t values[4] = { + accumulators[row][0], accumulators[row][1], + accumulators[row][2], accumulators[row][3] + }; + const float* residual = epilogue == NULL || epilogue->residual == NULL + ? NULL : epilogue->residual + output_base; + lw_wasm128_apply_epilogue_oc16(values, epilogue, oc, residual, 1); + for (uint32_t group = 0u; group < 4u; ++group) { + wasm_v128_store(output + output_base + group * 4u, values[group]); + } + } else { + float sums[LW_NHWC_OC_BLOCK]; + for (uint32_t group = 0u; group < 4u; ++group) { + wasm_v128_store(sums + group * 4u, accumulators[row][group]); + } + if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_GELU) { + for (uint32_t lane = 0u; lane < lanes; ++lane) { + if (epilogue->post_bias != NULL) + sums[lane] += epilogue->post_bias[oc + lane]; + if (epilogue->residual != NULL) + sums[lane] += epilogue->residual[output_base + lane]; + } + lw_wasm128_gelu_f32(sums, sums, LW_NHWC_OC_BLOCK); + for (uint32_t lane = 0u; lane < lanes; ++lane) + output[output_base + lane] = sums[lane]; + continue; + } + for (uint32_t lane = 0u; lane < lanes; ++lane) { + float value = sums[lane]; + if (epilogue != NULL) { + if (epilogue->post_bias != NULL) { + value += epilogue->post_bias[oc + lane]; + } + if (epilogue->residual != NULL) { + value += epilogue->residual[output_base + lane]; + } + value = apply_activation(value, epilogue->activation); + } + output[output_base + lane] = value; + } + } + } + } + } +} + +static inline void wasm_pointwise_store_row( + v128_t value0, v128_t value1, v128_t value2, v128_t value3, + const lw_nhwc_epilogue* epilogue, float* output, size_t output_base, + uint32_t oc, uint32_t lanes) { + if (lanes == LW_NHWC_OC_BLOCK) { + v128_t values[4] = {value0, value1, value2, value3}; + const float* residual = epilogue == NULL || epilogue->residual == NULL + ? NULL : epilogue->residual + output_base; + lw_wasm128_apply_epilogue_oc16(values, epilogue, oc, residual, 1); + wasm_v128_store(output + output_base, values[0]); + wasm_v128_store(output + output_base + 4u, values[1]); + wasm_v128_store(output + output_base + 8u, values[2]); + wasm_v128_store(output + output_base + 12u, values[3]); + return; + } + { + float sums[LW_NHWC_OC_BLOCK]; + wasm_v128_store(sums, value0); + wasm_v128_store(sums + 4u, value1); + wasm_v128_store(sums + 8u, value2); + wasm_v128_store(sums + 12u, value3); + if (epilogue != NULL && epilogue->activation == LW_NHWC_ACT_GELU) { + for (uint32_t lane = 0u; lane < lanes; ++lane) { + if (epilogue->post_bias != NULL) + sums[lane] += epilogue->post_bias[oc + lane]; + if (epilogue->residual != NULL) + sums[lane] += epilogue->residual[output_base + lane]; + } + lw_wasm128_gelu_f32(sums, sums, LW_NHWC_OC_BLOCK); + for (uint32_t lane = 0u; lane < lanes; ++lane) + output[output_base + lane] = sums[lane]; + return; + } + for (uint32_t lane = 0u; lane < lanes; ++lane) { + float value = sums[lane]; + if (epilogue != NULL) { + if (epilogue->post_bias != NULL) + value += epilogue->post_bias[oc + lane]; + if (epilogue->residual != NULL) + value += epilogue->residual[output_base + lane]; + value = apply_activation(value, epilogue->activation); + } + output[output_base + lane] = value; + } + } +} + +/* Two pixels share each OC16 weight block while keeping only eight SIMD + * accumulators live. The IC accumulation order and non-FMA mul/add sequence + * are identical to the existing 4x16 kernel. */ +void lw_wasm128_nhwc_pointwise_2x16_f32(const float* input, + const float* packed_weights, + const lw_nhwc_epilogue* epilogue, + float* output, uint32_t pixels, + uint32_t input_channels, + uint32_t output_channels) { + uint32_t pixel = 0u; + for (; pixel + 1u < pixels; pixel += 2u) { + const float* input0 = input + (size_t)pixel * input_channels; + const float* input1 = input + (size_t)(pixel + 1u) * input_channels; + for (uint32_t oc = 0u; oc < output_channels; oc += LW_NHWC_OC_BLOCK) { + uint32_t lanes = output_channels - oc; + v128_t bias0, bias1, bias2, bias3; + const float* weights = packed_weights + + (size_t)(oc / LW_NHWC_OC_BLOCK) * input_channels * LW_NHWC_OC_BLOCK; + if (lanes > LW_NHWC_OC_BLOCK) lanes = LW_NHWC_OC_BLOCK; + if (lanes == LW_NHWC_OC_BLOCK && epilogue != NULL && + epilogue->bias != NULL) { + bias0 = wasm_v128_load(epilogue->bias + oc); + bias1 = wasm_v128_load(epilogue->bias + oc + 4u); + bias2 = wasm_v128_load(epilogue->bias + oc + 8u); + bias3 = wasm_v128_load(epilogue->bias + oc + 12u); + } else if (epilogue != NULL && epilogue->bias != NULL) { + float tail_bias[LW_NHWC_OC_BLOCK] = {0.0f}; + for (uint32_t lane = 0u; lane < lanes; ++lane) + tail_bias[lane] = epilogue->bias[oc + lane]; + bias0 = wasm_v128_load(tail_bias); + bias1 = wasm_v128_load(tail_bias + 4u); + bias2 = wasm_v128_load(tail_bias + 8u); + bias3 = wasm_v128_load(tail_bias + 12u); + } else { + const v128_t zero = wasm_f32x4_splat(0.0f); + bias0 = zero; + bias1 = zero; + bias2 = zero; + bias3 = zero; + } + + v128_t a00 = bias0, a01 = bias1, a02 = bias2, a03 = bias3; + v128_t a10 = bias0, a11 = bias1, a12 = bias2, a13 = bias3; + for (uint32_t ic = 0u; ic < input_channels; ++ic) { + const float* packed = weights + (size_t)ic * LW_NHWC_OC_BLOCK; + const v128_t x0 = wasm_f32x4_splat(input0[ic]); + const v128_t x1 = wasm_f32x4_splat(input1[ic]); + { + const v128_t w = wasm_v128_load(packed); + a00 = wasm_f32x4_add(a00, wasm_f32x4_mul(x0, w)); + a10 = wasm_f32x4_add(a10, wasm_f32x4_mul(x1, w)); + } + { + const v128_t w = wasm_v128_load(packed + 4u); + a01 = wasm_f32x4_add(a01, wasm_f32x4_mul(x0, w)); + a11 = wasm_f32x4_add(a11, wasm_f32x4_mul(x1, w)); + } + { + const v128_t w = wasm_v128_load(packed + 8u); + a02 = wasm_f32x4_add(a02, wasm_f32x4_mul(x0, w)); + a12 = wasm_f32x4_add(a12, wasm_f32x4_mul(x1, w)); + } + { + const v128_t w = wasm_v128_load(packed + 12u); + a03 = wasm_f32x4_add(a03, wasm_f32x4_mul(x0, w)); + a13 = wasm_f32x4_add(a13, wasm_f32x4_mul(x1, w)); + } + } + wasm_pointwise_store_row(a00, a01, a02, a03, epilogue, output, + (size_t)pixel * output_channels + oc, oc, lanes); + wasm_pointwise_store_row(a10, a11, a12, a13, epilogue, output, + (size_t)(pixel + 1u) * output_channels + oc, + oc, lanes); + } + } + if (pixel < pixels) { + lw_nhwc_epilogue tail_epilogue; + const lw_nhwc_epilogue* tail_epilogue_ptr = NULL; + if (epilogue != NULL) { + tail_epilogue = *epilogue; + if (tail_epilogue.residual != NULL) + tail_epilogue.residual += (size_t)pixel * output_channels; + tail_epilogue_ptr = &tail_epilogue; + } + lw_wasm128_nhwc_pointwise_4x16_f32( + input + (size_t)pixel * input_channels, packed_weights, + tail_epilogue_ptr, output + (size_t)pixel * output_channels, + 1u, input_channels, output_channels); + } +} diff --git a/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_packed_matmul.c b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_packed_matmul.c new file mode 100644 index 00000000..2d0be2c3 --- /dev/null +++ b/deploy/ocr/vendor/lw.PPOCR.C/src/simd/wasm128_packed_matmul.c @@ -0,0 +1,123 @@ +#include "../kernels/packed_matmul_internal.h" + +#include +#include +#include + +#if defined(__wasm_simd128__) +#include +#endif + +#define LW_CTC_PANEL LW_PACKED_MATMUL_COLUMN_TILE +#define LW_CTC_ROWS 4u + +/* The packed layout is [panel][inner][16]. Keep panels outermost so their + * weights stay cache-resident across row tiles, without a logits tensor. */ +static void project_panel(const float* input, const float* packed_weights, + const float* bias, float values[LW_CTC_ROWS][LW_CTC_PANEL], + uint32_t rows, uint32_t inner_dimension, + uint32_t column_base, uint32_t valid_columns) { + const float* panel = packed_weights + + (size_t)(column_base / LW_CTC_PANEL) * inner_dimension * LW_CTC_PANEL; +#if defined(__wasm_simd128__) + v128_t accumulators[LW_CTC_ROWS][4]; + for (uint32_t row = 0u; row < rows; ++row) { + for (uint32_t group = 0u; group < 4u; ++group) { + accumulators[row][group] = wasm_f32x4_splat(0.0f); + } + } + for (uint32_t inner = 0u; inner < inner_dimension; ++inner) { + v128_t weights[4]; + const float* packed = panel + (size_t)inner * LW_CTC_PANEL; + for (uint32_t group = 0u; group < 4u; ++group) { + weights[group] = wasm_v128_load(packed + group * 4u); + } + for (uint32_t row = 0u; row < rows; ++row) { + v128_t value = wasm_f32x4_splat(input[(size_t)row * inner_dimension + inner]); + for (uint32_t group = 0u; group < 4u; ++group) { + accumulators[row][group] = wasm_f32x4_add( + accumulators[row][group], wasm_f32x4_mul(value, weights[group])); + } + } + } + for (uint32_t row = 0u; row < rows; ++row) { + for (uint32_t group = 0u; group < 4u; ++group) { + wasm_v128_store(values[row] + group * 4u, accumulators[row][group]); + } + } +#else + for (uint32_t row = 0u; row < rows; ++row) { + for (uint32_t lane = 0u; lane < valid_columns; ++lane) { + float sum = 0.0f; + for (uint32_t inner = 0u; inner < inner_dimension; ++inner) { + sum += input[(size_t)row * inner_dimension + inner] * + panel[(size_t)inner * LW_CTC_PANEL + lane]; + } + values[row][lane] = sum; + } + } +#endif + for (uint32_t row = 0u; row < rows; ++row) { + for (uint32_t lane = 0u; lane < valid_columns; ++lane) { + values[row][lane] += bias != NULL ? bias[column_base + lane] : 0.0f; + } + } +} + +void lw_wasm128_packed_matmul_argmax_scores_f32( + const float* input, const float* packed_weights, const float* bias, + uint32_t* best_indices, float* scores, uint32_t rows, + uint32_t inner_dimension, uint32_t columns) { + if (columns == 0u) return; + for (uint32_t row = 0u; row < rows; ++row) { + scores[row] = -INFINITY; + best_indices[row] = 0u; + } + for (uint32_t column_base = 0u; column_base < columns; + column_base += LW_CTC_PANEL) { + uint32_t valid_columns = columns - column_base; + if (valid_columns > LW_CTC_PANEL) valid_columns = LW_CTC_PANEL; + for (uint32_t row_base = 0u; row_base < rows; row_base += LW_CTC_ROWS) { + float values[LW_CTC_ROWS][LW_CTC_PANEL]; + uint32_t tile_rows = rows - row_base; + if (tile_rows > LW_CTC_ROWS) tile_rows = LW_CTC_ROWS; + project_panel(input + (size_t)row_base * inner_dimension, + packed_weights, bias, values, tile_rows, + inner_dimension, column_base, valid_columns); + for (uint32_t row = 0u; row < tile_rows; ++row) { + for (uint32_t lane = 0u; lane < valid_columns; ++lane) { + float value = values[row][lane]; + if (value > scores[row_base + row]) { + scores[row_base + row] = value; + best_indices[row_base + row] = column_base + lane; + } + } + } + } + } +} + +void lw_wasm128_ctc_row_probabilities_f32( + const float* input, const float* packed_weights, const float* bias, + const uint32_t* best_indices, const float* scores, float* probabilities, + uint32_t rows, uint32_t inner_dimension, uint32_t columns) { + for (uint32_t row = 0u; row < rows; ++row) { + uint32_t index = best_indices[row]; + if (index != 0u && (row == 0u || index != best_indices[row - 1u])) { + float sum = 0.0f; + for (uint32_t column_base = 0u; column_base < columns; + column_base += LW_CTC_PANEL) { + float values[LW_CTC_ROWS][LW_CTC_PANEL]; + uint32_t valid_columns = columns - column_base; + if (valid_columns > LW_CTC_PANEL) valid_columns = LW_CTC_PANEL; + project_panel(input + (size_t)row * inner_dimension, + packed_weights, bias, values, 1u, + inner_dimension, column_base, valid_columns); + for (uint32_t lane = 0u; lane < valid_columns; ++lane) { + sum += expf(values[0][lane] - scores[row]); + } + } + probabilities[row] = 1.0f / sum; + } + } +} diff --git a/frontend/src/features/admin/views/AdminSystemConfigsView.vue b/frontend/src/features/admin/views/AdminSystemConfigsView.vue index 701e0564..a4843aa6 100644 --- a/frontend/src/features/admin/views/AdminSystemConfigsView.vue +++ b/frontend/src/features/admin/views/AdminSystemConfigsView.vue @@ -1043,19 +1043,19 @@ function formatConfigValue(row: SystemConfig) {
-

PaddleOCR

+

OCR

二维码群名识别配置

- 配置企业微信群二维码上传时使用的 OCR 识别接口,保存后无需重新构建前端。 + 默认使用私有本地 OCR sidecar;以下远程 PaddleOCR 配置仅作为备用通道。
{{ paddleOcrTokenConfig.value.trim() ? '已配置' : '未配置' }} - API Token + 远程备用 Token 编辑
- 接口 + 远程备用接口 {{ paddleOcrJobUrlConfig.value || '未配置' }} 编辑
@@ -1065,8 +1065,8 @@ function formatConfigValue(row: SystemConfig) { 编辑
- 直连 - 二维码池上传时调用 + 本地优先 + 本地失败后可手动填写
更新 diff --git a/scripts/deploy-prod.sh b/scripts/deploy-prod.sh index ac964ac1..bb8b55fb 100755 --- a/scripts/deploy-prod.sh +++ b/scripts/deploy-prod.sh @@ -765,6 +765,7 @@ main() { wait_service_healthy mysql "MySQL" wait_service_healthy redis "Redis" + wait_service_healthy ocr "OCR sidecar" wait_service_healthy caddy "Caddy" ensure_mysql_backup_user reset_database diff --git a/scripts/dev.sh b/scripts/dev.sh index 8e55759f..dd8fe705 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -26,6 +26,7 @@ MYSQL_PORT="${DEV_MYSQL_PORT:-13306}" REDIS_PORT="${DEV_REDIS_PORT:-16379}" MINIO_API_PORT="${DEV_MINIO_API_PORT:-19090}" MINIO_CONSOLE_PORT="${DEV_MINIO_CONSOLE_PORT:-19091}" +OCR_PORT="${DEV_OCR_PORT:-18787}" BACKEND_HEALTH_URL="${DEV_BACKEND_HEALTH_URL:-http://127.0.0.1:${BACKEND_PORT}/health}" FRONTEND_HEALTH_URL="${DEV_FRONTEND_HEALTH_URL:-http://127.0.0.1:${FRONTEND_PORT}/}" DISTRIBUTION_HEALTH_URL="${DEV_DISTRIBUTION_HEALTH_URL:-http://127.0.0.1:${DISTRIBUTION_PORT}/}" @@ -107,6 +108,8 @@ show_help() { MinIO API 宿主机映射端口 DEV_MINIO_CONSOLE_PORT=19091 MinIO 控制台宿主机映射端口 + DEV_OCR_PORT=18787 + OCR sidecar 宿主机映射端口 DEV_BACKEND_HEALTH_URL=http://127.0.0.1:18088/health 后端健康检查地址 DEV_FRONTEND_HEALTH_URL=http://127.0.0.1:5189/ @@ -527,6 +530,7 @@ start_backend() { export MYSQL_DSN="hfb:secret@tcp(127.0.0.1:${MYSQL_PORT})/${MYSQL_DATABASE}?charset=utf8mb4&parseTime=True&loc=Local" export REDIS_ADDR="127.0.0.1:${REDIS_PORT}" export STORAGE_ENDPOINT="http://localhost:${MINIO_API_PORT}" + export OCR_SERVICE_URL="${OCR_SERVICE_URL:-http://127.0.0.1:${OCR_PORT}}" go run ./cmd/api ) & BACKEND_PID="$!" @@ -715,17 +719,20 @@ main() { ensure_port_free "${REDIS_PORT}" "Redis" hfb-redis ensure_port_free "${MINIO_API_PORT}" "MinIO API" hfb-minio ensure_port_free "${MINIO_CONSOLE_PORT}" "MinIO 控制台" hfb-minio + ensure_port_free "${OCR_PORT}" "OCR sidecar" hfb-ocr - log "启动开发依赖:MySQL、Redis、MinIO" + log "启动开发依赖:MySQL、Redis、MinIO、OCR sidecar" export DEV_MYSQL_PORT="${MYSQL_PORT}" export DEV_REDIS_PORT="${REDIS_PORT}" export DEV_MINIO_API_PORT="${MINIO_API_PORT}" export DEV_MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT}" + export DEV_OCR_PORT="${OCR_PORT}" docker compose -f "${COMPOSE_FILE}" up -d wait_container_healthy hfb-mysql "MySQL" wait_container_healthy hfb-redis "Redis" wait_container_healthy hfb-minio "MinIO" + wait_container_healthy hfb-ocr "OCR sidecar" reset_database run_migrations