{ "$schema": "https://meyorda.uz/research/uzbek-food-ai-benchmark-2026/protocol.schema.json", "$id": "https://meyorda.uz/research/uzbek-food-ai-benchmark-2026/protocol-v0.1.json", "protocolVersion": "0.1.0", "status": "protocol_published_before_results", "resultsPublished": false, "publishedAt": "2026-08-24", "lastModified": "2026-08-24", "titles": { "ru": "Uzbek Food AI Benchmark 2026: протокол оценки распознавания блюд по фото", "uz": "Uzbek Food AI Benchmark 2026: taomlarni suratdan aniqlashni baholash protokoli", "en": "Uzbek Food AI Benchmark 2026: food-photo recognition evaluation protocol" }, "abstracts": { "ru": "Предварительно зафиксированный план открытого сравнительного теста AI-сервисов на блюдах, распространённых в Узбекистане. Документ описывает выбор кейсов, референсные данные, метрики, правила сравнения и ограничения. Результаты в этой версии отсутствуют.", "uz": "O‘zbekistonda keng tarqalgan taomlarda AI-servislarni ochiq taqqoslash uchun oldindan belgilangan reja. Hujjat holatlarni tanlash, tayanch ma’lumotlar, mezonlar, taqqoslash qoidalari va cheklovlarni bayon qiladi. Bu versiyada natijalar yo‘q.", "en": "A pre-specified plan for an open comparison of consumer AI services on foods commonly eaten in Uzbekistan. It defines case selection, references, outcomes, comparison rules, and limitations. This version contains no results." }, "organizer": { "name": "Meyorda", "url": "https://meyorda.uz", "contact": "support@meyorda.uz", "conflictDisclosure": "Meyorda is both the organizer of the protocol and one of the systems planned for evaluation. The final report must disclose this relationship, publish all eligible cases and failures, and avoid an undisclosed composite winner score." }, "researchQuestion": { "ru": "Как доступные пользователю AI-сервисы распознают распространённые в Узбекистане блюда и оценивают видимую порцию по одной и той же фотографии?", "uz": "Foydalanuvchi uchun mavjud AI-servislar O‘zbekistonda keng tarqalgan taomlarni va ko‘rinadigan porsiyani bir xil surat asosida qanday baholaydi?", "en": "How do consumer-accessible AI services recognize foods commonly eaten in Uzbekistan and estimate the visible portion from the same photograph?" }, "scope": { "market": "Uzbekistan", "plannedDishTypes": 50, "plannedIndependentPlatesPerDish": 3, "plannedIndependentPlates": 150, "plannedImagesPerPlate": 2, "plannedNegativeControls": 10, "plannedSystems": 4, "primaryTrack": "photo_only", "languages": [ "uz", "ru" ] }, "systemEligibility": { "rosterStatus": "to_be_frozen_before_testing", "rules": [ "The service must be accessible to a consumer testing from Uzbekistan during the declared test window.", "The service must accept the same prepared image file or an operationally equivalent image input.", "The service must return at least a dish label or a nutrition estimate from a food photograph.", "The tested plan, platform, locale, version when visible, settings, and test timestamp must be recorded." ], "tentativeCandidates": [ "Meyorda", "YaYo AI", "Eda AI", "Cal AI" ], "candidateDisclaimer": "Tentative candidates are not the final roster. Inclusion does not imply availability, endorsement, partnership, or any result. The final roster must be frozen before scoring begins." }, "caseDesign": { "independentUnit": "plate", "plannedTracks": [ { "id": "controlled", "description": "A consistent full-plate smartphone photograph with recorded lighting, angle, and serving context." }, { "id": "everyday", "description": "A second smartphone photograph of the same plate under a documented everyday viewing condition." } ], "dishRosterRule": "The bilingual dish roster and acceptable family aliases must be frozen before system runs. Hidden variants are scored separately only when the reference label is reasonably observable from the image.", "imageRightsRule": "Only purpose-collected images with documented permission for publication may enter the public release. Personal identifiers and location metadata must be removed.", "negativeControlRule": "Non-food or empty-serving controls are reported separately and never included in dish-recognition accuracy." }, "referenceMethods": { "dishIdentity": "Two bilingual reviewers use a frozen RU/UZ alias guide. Disagreements are retained and resolved by documented adjudication before systems are unblinded in the scoring table.", "portion": "The edible serving is weighed on a tared kitchen scale. Scale precision and measurement notes are recorded per plate.", "nutrition": "Calorie and macro error is evaluated only on a declared recipe-grounded subset with ingredient weights and final cooked yield. Values remain composition-table reference estimates, not laboratory measurements.", "restaurantRule": "A restaurant plate without a documented recipe may be scored for dish identity and measured portion, but not for calorie accuracy merely from its menu name or appearance." }, "outcomes": [ { "id": "dish_family_accuracy", "kind": "primary", "unit": "proportion", "description": "Share of independent plates assigned to the frozen acceptable dish family." }, { "id": "dish_subtype_accuracy", "kind": "secondary", "unit": "proportion", "description": "Share of eligible plates assigned to the correct visible subtype; hidden or visually indeterminate variants are excluded." }, { "id": "portion_absolute_error", "kind": "primary", "unit": "grams", "description": "Median absolute difference between predicted and measured edible portion." }, { "id": "portion_absolute_percentage_error", "kind": "secondary", "unit": "percent", "description": "Median absolute percentage error for measured edible portions." }, { "id": "calorie_absolute_error", "kind": "secondary", "unit": "kilocalories", "description": "Absolute error against the recipe-grounded reference subset only." }, { "id": "interval_coverage", "kind": "secondary", "unit": "proportion", "description": "Share of eligible reference values contained within a system-provided interval, reported together with interval width." }, { "id": "abstention_and_missing_output", "kind": "safety", "unit": "proportion", "description": "Uncertain, refused, unavailable, and missing outputs are reported as distinct categories and never converted to zero." } ], "reportingRules": [ "Plate, not photograph, is the independent statistical unit.", "Report both overall results and equal-dish-weighted summaries so repeated variants do not dominate the comparison.", "Report uncertainty intervals clustered by independent plate or dish as declared in the final analysis plan.", "Do not compare proprietary confidence numbers as though they share one calibrated scale.", "Do not publish an undisclosed composite winner score. Any later composite requires a protocol amendment made before results are inspected.", "Publish eligible failures, refusals, and missing outputs alongside successful cases.", "Use language such as reference estimate for recipe-derived nutrition and never laboratory ground truth." ], "reproducibility": { "inputNormalization": { "maximumEdgePixels": 1280, "format": "image/jpeg", "jpegQuality": 82, "chromaSubsampling": "4:2:0" }, "runRules": [ "Prepare one normalized benchmark image and provide that same file to every eligible system when technically possible.", "Run the primary track without dish-name descriptions or post-result corrections.", "Randomize case order and complete comparison runs within a declared, short test window.", "Record raw displayed values, screenshots where redistribution is permitted, timestamps, failures, and visible version information.", "Freeze the Meyorda code revision, model configuration, prompt revision, and scoring guide before evaluated runs." ], "plannedArtifacts": [ "case manifest", "data dictionary", "system manifest", "raw normalized outputs", "scoring decisions", "analysis code", "change log" ] }, "limitations": [ "A photograph cannot reveal exact oil, hidden filling, meat fat, recipe yield, or edible weight.", "Three plates per dish support a descriptive benchmark, not a definitive estimate for every regional recipe.", "Consumer applications and underlying models can change after the recorded test window.", "Meyorda organizes the comparison and is therefore not an independent evaluator.", "The public protocol does not make medical, laboratory-accuracy, superiority, or market-wide coverage claims." ], "changePolicy": { "beforeResults": "Material changes create a new numbered protocol version with a dated explanation.", "afterResults": "The frozen protocol remains available. Corrections to data or analysis create a versioned release and a public change log; results are not silently replaced." }, "artifacts": { "landingPageRu": "https://meyorda.uz/ru/research/uzbek-food-ai-benchmark-2026", "landingPageUz": "https://meyorda.uz/uz/research/uzbek-food-ai-benchmark-2026", "protocolJson": "https://meyorda.uz/research/uzbek-food-ai-benchmark-2026/protocol-v0.1.json", "schemaJson": "https://meyorda.uz/research/uzbek-food-ai-benchmark-2026/protocol.schema.json", "datasetStatus": "not_published", "doiStatus": "not_assigned", "licenseStatus": "not_assigned" } }