diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c73d242e..f45a5214 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,7 +229,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} slug: embedded-dev-research/ITLabAI - evaluate-model: + evaluate-model-alexnet: runs-on: ubuntu-latest needs: [build-linux] permissions: @@ -319,3 +319,285 @@ jobs: git commit -m "[CI] Update accuracy: $(cat accuracy_value.txt)" git push origin master fi + + evaluate-models-onnx: + runs-on: ubuntu-latest + needs: [build-linux] + permissions: + contents: write + strategy: + fail-fast: false + matrix: + model: [googlenet, densenet, resnet, yolo] + include: + - model: googlenet + parser: parser_onnx.py + model_file: GoogLeNet.onnx + model_path: docs/models/GoogLeNet.onnx + model_url: '' + extra_args: "--onednn 10000" + - model: densenet + parser: parser_onnx.py + model_file: densenet121_Opset16.onnx + model_url: https://github.com/onnx/models/raw/refs/heads/main/Computer_Vision/densenet121_Opset16_timm/densenet121_Opset16.onnx?download= + extra_args: "--onednn 1000" + - model: resnet + parser: parser_onnx.py + model_file: resnest101e_Opset16.onnx + model_url: https://github.com/onnx/models/raw/refs/heads/main/Computer_Vision/resnest101e_Opset16_timm/resnest101e_Opset16.onnx?download= + extra_args: "--onednn 1000" + - model: yolo + parser: parser_onnx.py + model_file: yolo11x-cls.pt + model_url: https://github.com/ultralytics/assets/releases/download/v8.4.0/yolo11x-cls.pt + extra_args: "--onednn 1000" + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download binary and libs + uses: actions/download-artifact@v4 + with: + name: mnist-RELEASE + path: build/ + + - name: Set binary path + id: set_eval_binary + run: | + echo "EVAL_BINARY=build/bin/ACC" >> $GITHUB_OUTPUT + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libgtk-3-0 libtbb12 libjpeg-dev libpng-dev libtiff-dev libopenjp2-7 libdnnl3 + sudo ldconfig + + - name: Download model (if URL provided) + if: matrix.model_url != '' + run: | + mkdir -p docs/models + echo "Скачивание ${{ matrix.model_file }} из ${{ matrix.model_url }}" + wget -O docs/models/${{ matrix.model_file }} ${{ matrix.model_url }} + ls -la docs/models/ + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install Python dependencies from requirements.txt + run: | + cd app/Converters + pip install -r requirements.txt + cd ../.. + + - name: Cache model JSON files + id: cache-model-json + uses: actions/cache@v4 + with: + path: docs/jsons + key: model-json-${{ matrix.model }}-${{ hashFiles('app/Converters/parser_onnx.py', 'app/Converters/requirements.txt') }} + restore-keys: | + model-json-${{ matrix.model }}- + model-json- + + - name: Generate model JSON + if: steps.cache-model-json.outputs.cache-hit != 'true' + run: | + mkdir -p docs/jsons + cd app/Converters + python ${{ matrix.parser }} ${{ matrix.model }} + cd ../.. + + - name: Cache ImageNet-Paste dataset + id: cache-imagenet-restore + uses: actions/cache/restore@v4 + with: + path: docs/ImageNet + key: imagenet-paste-${{ github.run_id }}-${{ hashFiles('app/Converters/download_imagenet.py') }} + restore-keys: | + imagenet-paste- + + - name: Download ImageNet-Paste dataset + if: steps.cache-imagenet-restore.outputs.cache-hit != 'true' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + mkdir -p docs/ImageNet/test + cd app/Converters + python download_imagenet.py + cd ../.. + + - name: Save ImageNet-Paste dataset + if: steps.cache-imagenet-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: docs/ImageNet + key: imagenet-paste-v1-${{ hashFiles('app/Converters/download_imagenet.py') }} + + - name: Prepare environment + run: | + chmod +x "${{ steps.set_eval_binary.outputs.EVAL_BINARY }}" + echo "LD_LIBRARY_PATH=$PWD/build/bin/all_libs:/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV + + - name: Increase system limits + run: | + sudo prlimit --pid $$ --as=unlimited + ulimit -s unlimited + ulimit -c unlimited + echo "Новые лимиты:" + ulimit -a + + - name: Run evaluation + run: | + DATASET_PATH="docs/ImageNet/test" + MODEL="${{ matrix.model }}" + EXTRA_ARGS="${{ matrix.extra_args }}" + + echo "Запуск оценки для модели $MODEL" + echo "Команда: ${{ steps.set_eval_binary.outputs.EVAL_BINARY }} --model $MODEL $EXTRA_ARGS" + + echo "Системная информация до запуска:" + echo "--- Память ---" + free -h + echo "--- CPU ---" + top -bn1 | head -5 + echo "--- Диск ---" + df -h docs/ImageNet/test/ + + TIMESTAMP=$(date +%s) + MONITOR_LOG="monitor_$MODEL.log" + + monitor_process() { + local pid=$1 + local log_file=$2 + + echo "Мониторинг процесса $pid..." + + while kill -0 $pid 2>/dev/null; do + NOW=$(date '+%H:%M:%S') + + MEM=$(ps -o rss= -p $pid 2>/dev/null | awk '{print $1/1024 " MB"}' || echo "N/A") + + CPU=$(ps -o pcpu= -p $pid 2>/dev/null || echo "N/A") + + TOTAL_MEM=$(free -m | awk 'NR==2{print $3 " MB / " $2 " MB"}') + + echo "[$NOW] PID: $pid | MEM: $MEM | CPU: $CPU% | System MEM: $TOTAL_MEM" >> $log_file + + sleep 2 + done + + echo "Процесс $pid завершен" >> $log_file + } + + echo "Запуск ACC в фоновом режиме..." + "${{ steps.set_eval_binary.outputs.EVAL_BINARY }}" \ + --model $MODEL \ + $EXTRA_ARGS > accuracy_$MODEL.txt 2>&1 & + ACC_PID=$! + + echo "PID процесса: $ACC_PID" + + monitor_process $ACC_PID $MONITOR_LOG & + MONITOR_PID=$! + + wait $ACC_PID + EXIT_CODE=$? + + kill $MONITOR_PID 2>/dev/null || true + + echo "Результаты мониторинга:" + if [ -f "$MONITOR_LOG" ]; then + cat "$MONITOR_LOG" + else + echo "Лог мониторинга не создан" + fi + + echo "Системная информация после запуска:" + free -h + + echo "Код завершения: $EXIT_CODE" + + if [ $EXIT_CODE -eq 143 ]; then + echo "Ошибка 143 (SIGTERM) - процесс убит системой" + + if sudo dmesg | tail -20 | grep -i "killed process" | grep -q "$ACC_PID"; then + echo "Подтверждено: процесс убит OOM Killer" + sudo dmesg | tail -20 | grep -i "killed process" | tail -5 + else + echo "Не найдено подтверждение OOM Killer в логах" + sudo dmesg | tail -20 + fi + + elif [ $EXIT_CODE -ne 0 ]; then + echo "Ошибка при оценке модели $MODEL (код: $EXIT_CODE)" + else + echo "Оценка успешно завершена" + fi + + echo "Лог ACC:" + if [ -f "accuracy_$MODEL.txt" ]; then + cat "accuracy_$MODEL.txt" + else + echo "Файл лога не создан" + fi + + # Выход с ошибкой если нужно + if [ $EXIT_CODE -ne 0 ]; then + exit $EXIT_CODE + fi + + - name: Extract accuracy value + run: | + ACCURACY=$(grep -oE '[0-9]+\.?[0-9]*%' accuracy_${{ matrix.model }}.txt | head -1 || echo "0%") + echo "$ACCURACY" > accuracy_value_${{ matrix.model }}.txt + echo "Accuracy for ${{ matrix.model }}: $ACCURACY" + + - name: Upload accuracy artifacts + uses: actions/upload-artifact@v4 + with: + name: accuracy-${{ matrix.model }} + path: | + accuracy_${{ matrix.model }}.txt + accuracy_value_${{ matrix.model }}.txt + + - name: Update README for model (master only) + if: github.ref == 'refs/heads/master' + run: | + TOP1_ACC=$(grep -oE 'Top-1 Accuracy: [0-9]+\.?[0-9]*%' accuracy_${{ matrix.model }}.txt | grep -oE '[0-9]+\.?[0-9]*') + TOP5_ACC=$(grep -oE 'Top-5 Accuracy: [0-9]+\.?[0-9]*%' accuracy_${{ matrix.model }}.txt | grep -oE '[0-9]+\.?[0-9]*') + DATE=$(date '+%Y-%m-%d') + + if [ -z "$TOP1_ACC" ] || [ -z "$TOP5_ACC" ]; then + echo "Ошибка: Не удалось извлечь точность из файла accuracy_${{ matrix.model }}.txt" + cat accuracy_${{ matrix.model }}.txt + exit 1 + fi + + UPDATE_TEXT="Accuracy: Top-1: ${TOP1_ACC}% | Top-5: ${TOP5_ACC}% (updated: ${DATE})" + + if grep -q "" README.md; then + sed -i "s|.*|${UPDATE_TEXT}|" README.md + echo "Обновлена точность для ${{ matrix.model }} в README" + else + echo "Ошибка: Плейсхолдер не найден в README.md" + echo "Содержимое README.md:" + cat README.md + exit 1 + fi + + - name: Commit and push changes (main only) + if: github.ref == 'refs/heads/main' + run: | + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + git add README.md + if git diff-index --quiet HEAD --; then + echo "No changes to commit" + else + git commit -m "[CI] Update accuracy for ${{ matrix.model }}: $(cat accuracy_value_${{ matrix.model }}.txt)" + git push origin master + fi diff --git a/README.md b/README.md index 853a75c7..c428b2dd 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,21 @@ # AlexNet-MNIST-Inference ## Model Performance -Accuracy: Stat: 98.01% (updated: 2025-04-28) +### AlexNet-MNIST Accuracy + + +### GoogLeNet Accuracy + + +### DenseNet Accuracy + + +### ResNet Accuracy + + +### YOLO Accuracy + + ## Short description A lightweight C++ library for performing high-performance inference on classification tasks. Designed for efficiency and educational purposes, this project demonstrates how classic CNNs can be optimized for small-scale tasks in native environments. ### Key Features: diff --git a/app/Converters/download_imagenet.py b/app/Converters/download_imagenet.py new file mode 100644 index 00000000..c6d9d1a8 --- /dev/null +++ b/app/Converters/download_imagenet.py @@ -0,0 +1,110 @@ +import os +from datasets import load_dataset +from huggingface_hub import login +from PIL import Image +from collections import defaultdict + +hf_token = os.environ.get('HF_TOKEN') +if hf_token: + print("Авторизация на Hugging Face Hub...") + login(token=hf_token) +else: + print("Внимание: HF_TOKEN не найден, могут быть ограничения rate limiting") + +base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +output_dir = os.path.join(base_dir, 'docs', 'ImageNet', 'test') +os.makedirs(output_dir, exist_ok=True) + +print("Загрузка датасета helenqu/ImageNet-Paste...") +ds = load_dataset( + "helenqu/ImageNet-Paste", + split="validation", + token=hf_token +) + +print(f"Датасет загружен. Всего записей: {len(ds)}") + +print(f"Ключи первой записи: {ds[0].keys()}") +if 'label' in ds[0]: + print(f"Пример метки: {ds[0]['label']}") + print(f"Тип метки: {type(ds[0]['label'])}") + +print("\nСоздание папок для классов 00000-00999 в test/...") +for i in range(1000): + class_folder = os.path.join(output_dir, f"{i:05d}") + os.makedirs(class_folder, exist_ok=True) +print("Папки созданы!") + +counters = defaultdict(int) + +total_images = len(ds) +images_per_class = 50 +max_images = 1000 * images_per_class + +print(f"\nНачинаем сохранение {min(total_images, max_images)} изображений...") +print(f"По {images_per_class} изображений в каждой из 1000 папок") +print(f"Путь: {output_dir}/00000/ ... /00999/") + +saved_count = 0 +skipped_count = 0 + +for i, item in enumerate(ds): + if saved_count >= max_images: + print(f"\nДостигнут лимит в {max_images} изображений") + break + + try: + image = item['image'] + if 'label' in item: + class_id = item['label'] + elif 'labels' in item: + class_id = item['labels'] + else: + skipped_count += 1 + if skipped_count % 100 == 0: + print(f"Пропущено {skipped_count} изображений: нет метки класса") + continue + + if not isinstance(class_id, (int, float)) or class_id < 0 or class_id >= 1000: + skipped_count += 1 + if skipped_count % 100 == 0: + print(f"Пропущено {skipped_count} изображений: некорректный class_id {class_id}") + continue + + class_id_int = int(class_id) + + if counters[class_id_int] >= images_per_class: + continue + + class_folder = os.path.join(output_dir, f"{class_id_int:05d}") + filename = f"image_{counters[class_id_int]}.jpg" + output_path = os.path.join(class_folder, filename) + + image.save(output_path, 'JPEG') + counters[class_id_int] += 1 + saved_count += 1 + + except Exception as e: + print(f"Ошибка при сохранении изображения {i}: {e}") + continue + +print(f"\n{'=' * 50}") +print(f"ГОТОВО!") +print(f"{'=' * 50}") +print(f"Всего сохранено: {saved_count} изображений") +print(f"Пропущено: {skipped_count} изображений") +print(f"Распределение по первым 10 классам:") +for class_id in range(10): + print(f" Класс {class_id:05d}: {counters[class_id]} изображений") + +classes_with_50 = sum(1 for c in range(1000) if counters[c] == 50) +print(f"\nКлассов с ровно 50 изображениями: {classes_with_50}/1000") + +if classes_with_50 < 1000: + print("\nКлассы с недостаточным количеством:") + for class_id in range(1000): + if counters[class_id] < 50 and counters[class_id] > 0: + print(f" Класс {class_id:05d}: только {counters[class_id]} изображений") + +print(f"\nПуть к данным: {output_dir}") +print(f"Пример: {output_dir}/00042/image_0.jpg") \ No newline at end of file diff --git a/app/Converters/parser_onnx.py b/app/Converters/parser_onnx.py index 33b20fef..8fcecbcb 100644 --- a/app/Converters/parser_onnx.py +++ b/app/Converters/parser_onnx.py @@ -1,9 +1,15 @@ +#!/usr/bin/env python3 import json -import onnx import os +import sys +import argparse +import onnx +import numpy as np from onnx import TensorProto from onnx import helper, numpy_helper from ultralytics import YOLO +import tensorflow as tf +from tensorflow.keras.models import load_model def convert_pt_to_onnx(pt_model_path, onnx_model_path=None): @@ -163,10 +169,38 @@ def default(self, obj): print(f"Модель успешно сохранена в {output_json_path}") +parser = argparse.ArgumentParser(description='Конвертация моделей в JSON формат') +parser.add_argument('model_name', type=str, + choices=['googlenet', 'densenet', 'resnet', 'yolo'], + help='Имя модели для обработки') + +args = parser.parse_args() BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +MODELS_DIR = os.path.join(BASE_DIR, 'docs', 'models') +JSONS_DIR = os.path.join(BASE_DIR, 'docs', 'jsons') + +os.makedirs(MODELS_DIR, exist_ok=True) +os.makedirs(JSONS_DIR, exist_ok=True) + +model_files = { + 'googlenet': 'GoogLeNet.onnx', + 'densenet': 'densenet121_Opset16.onnx', + 'resnet': 'resnest101e_Opset16.onnx', + 'yolo': 'yolo11x-cls.pt' +} + +output_files = { + 'googlenet': 'googlenet_onnx_model.json', + 'densenet': 'densenet121_Opset16_onnx_model.json', + 'resnet': 'resnest101e_Opset16_onnx_model.json', + 'yolo': 'yolo11x-cls_onnx_model.json', +} + +model_filename = model_files[args.model_name] +output_filename = output_files[args.model_name] -MODEL_PATH = os.path.join(BASE_DIR, 'docs\\models', 'resnest101e_Opset16.onnx') -MODEL_DATA_PATH = os.path.join(BASE_DIR, 'docs\\jsons', 'resnest101e_Opset16_onnx_model.json') +model_path = os.path.join(MODELS_DIR, model_filename) +output_path = os.path.join(JSONS_DIR, output_filename) -onnx_to_json(MODEL_PATH, MODEL_DATA_PATH) \ No newline at end of file +onnx_to_json(model_path, output_path) diff --git a/app/Converters/requirements.txt b/app/Converters/requirements.txt index 7c48b536..622b0e7d 100644 --- a/app/Converters/requirements.txt +++ b/app/Converters/requirements.txt @@ -5,5 +5,7 @@ onnx>=1.15.0 torch==2.2.1+cpu torchvision==0.17.1+cpu ultralytics>=8.0.0 -numpy>=1.21.0 -protobuf>=3.20.0 \ No newline at end of file +numpy>=1.21.0,<2.0.0 +protobuf>=3.20.0 +datasets>=2.14.0 +Pillow>=10.0.0 \ No newline at end of file diff --git a/app/Graph/acc_check.cpp b/app/Graph/acc_check.cpp index a864c36d..d37d4f89 100644 --- a/app/Graph/acc_check.cpp +++ b/app/Graph/acc_check.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -13,6 +15,8 @@ using namespace it_lab_ai; int main(int argc, char* argv[]) { std::string model_name = "alexnet_mnist"; RuntimeOptions options; + size_t num_photo = 1000; + size_t batch_size = 32; for (int i = 1; i < argc; ++i) { if (std::string(argv[i]) == "--model" && i + 1 < argc) { @@ -48,6 +52,18 @@ int main(int argc, char* argv[]) { } } else if (std::string(argv[i]) == "--threads" && i + 1 < argc) { options.threads = std::stoi(argv[++i]); + } else { + try { + num_photo = std::stoi(argv[i]); + + if (num_photo < 1 || num_photo > 50000) { + std::cerr << "Warning: num_photo should be between 1 and 10000 " + << "Using value: " << num_photo << '\n'; + } + } catch (const std::exception& e) { + std::cerr << "Error: Invalid numeric argument: " << argv[i] + << ". Using default value: 1000" << e.what() << '\n'; + } } } @@ -70,7 +86,6 @@ int main(int argc, char* argv[]) { size_t sum = std::accumulate(counts.begin(), counts.end(), size_t{0}); int count_pic = static_cast(sum) + 10; std::vector res(count_pic * 28 * 28); - Tensor input; Shape sh1({1, 5, 5, 3}); std::vector vec; vec.reserve(75); @@ -105,7 +120,7 @@ int main(int argc, char* argv[]) { } Shape sh({static_cast(count_pic), 1, 28, 28}); Tensor t = make_tensor(res, sh); - input = t; + Tensor input = t; Graph graph; build_graph_linear(graph, input, output, options, false); graph.inference(options); @@ -134,14 +149,13 @@ int main(int argc, char* argv[]) { << "%" << '\n'; return 0; } - std::vector counts; + + std::vector counts(1000, 0); std::vector image_paths; std::vector true_labels; std::vector all_image_data; size_t total_images = 0; - counts.resize(1000, 0); - for (int class_id = 0; class_id < 1000; ++class_id) { std::ostringstream folder_oss; folder_oss << std::setw(5) << std::setfill('0') << class_id; @@ -153,33 +167,45 @@ int main(int argc, char* argv[]) { entry.path().extension() == ".jpg" || entry.path().extension() == ".jpeg") { counts[class_id]++; - total_images++; } } } } - if (total_images == 0) { - std::cerr << "No images found in dataset path: " << dataset_path << '\n'; - return 1; - } + size_t images_per_class_base = num_photo / 1000; + size_t remaining = num_photo % 1000; int channels = input_shape[1]; int height = input_shape[2]; int width = input_shape[3]; size_t image_size = channels * height * width; + size_t output_classes = 1000; + + all_image_data.reserve(num_photo * image_size); + image_paths.reserve(num_photo); + true_labels.reserve(num_photo); - all_image_data.resize(total_images * image_size); + total_images = 0; - size_t current_index = 0; for (int class_id = 0; class_id < 1000; ++class_id) { + size_t need_from_class = images_per_class_base; + if (remaining > 0) { + need_from_class++; + remaining--; + } + + if (need_from_class == 0) continue; + std::ostringstream folder_oss; folder_oss << std::setw(5) << std::setfill('0') << class_id; std::string class_folder_path = dataset_path + "/" + folder_oss.str(); if (!fs::exists(class_folder_path)) continue; + size_t taken = 0; for (const auto& entry : fs::directory_iterator(class_folder_path)) { + if (taken >= need_from_class) break; + if (entry.path().extension() == ".png" || entry.path().extension() == ".jpg" || entry.path().extension() == ".jpeg") { @@ -194,82 +220,169 @@ int main(int argc, char* argv[]) { prepare_image(image, input_shape, model_name); const std::vector& image_data = *prepared_tensor.as(); - std::copy(image_data.begin(), image_data.end(), - all_image_data.begin() + current_index * image_size); + all_image_data.insert(all_image_data.end(), image_data.begin(), + image_data.end()); image_paths.push_back(entry.path().string()); true_labels.push_back(class_id); - current_index++; + taken++; + total_images++; } } - } - - it_lab_ai::Shape input_shape_imagenet( - {total_images, static_cast(channels), static_cast(height), - static_cast(width)}); - it_lab_ai::Tensor input = - it_lab_ai::make_tensor(all_image_data, input_shape_imagenet); - size_t output_classes = 1000; - it_lab_ai::Shape output_shape({total_images, output_classes}); - it_lab_ai::Tensor output = - it_lab_ai::Tensor(output_shape, it_lab_ai::Type::kFloat); + if (taken < need_from_class) { + std::cout << "Warning: Class " << class_id << " has only " << taken + << " images (needed " << need_from_class << ")" << '\n'; + } + } - Graph graph; - build_graph(graph, input, output, json_path, options, false); - graph.inference(options); - print_time_stats(graph); - std::vector> processed_outputs; - const std::vector& raw_output = *output.as(); - - for (size_t i = 0; i < total_images; ++i) { - std::vector single_output( - raw_output.begin() + i * output_classes, - raw_output.begin() + (i + 1) * output_classes); - std::vector processed_output = - process_model_output(single_output, model_name); - processed_outputs.push_back(processed_output); + if (total_images != num_photo) { + std::cout << "Warning: Requested " << num_photo << " images but loaded " + << total_images << " due to insufficient data" << '\n'; + num_photo = total_images; } int correct_predictions_top1 = 0; int correct_predictions_top5 = 0; - for (size_t i = 0; i < processed_outputs.size(); ++i) { - int true_label = true_labels[i]; - const std::vector& probabilities = processed_outputs[i]; - - std::vector indices(probabilities.size()); - std::iota(indices.begin(), indices.end(), 0); - std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) { - return probabilities[a] > probabilities[b]; - }); - - size_t predicted_class_top1 = indices[0]; - if (predicted_class_top1 == static_cast(true_label)) { - correct_predictions_top1++; + + it_lab_ai::Shape full_shape({num_photo, static_cast(channels), + static_cast(height), + static_cast(width)}); + it_lab_ai::Tensor dummy_input = make_tensor(all_image_data, full_shape); + + it_lab_ai::Shape full_output_shape({num_photo, output_classes}); + it_lab_ai::Tensor dummy_output(full_output_shape, it_lab_ai::Type::kFloat); + + Graph graph; + build_graph(graph, dummy_input, dummy_output, json_path, options, false); + + std::shared_ptr input_layer = nullptr; + std::shared_ptr output_layer = nullptr; + + for (int i = 0; i < graph.getLayersCount(); ++i) { + auto layer = graph.getLayerFromID(i); + if (layer->getName() == kInput) { + input_layer = layer; } + if (i == graph.getLayersCount() - 1) { + output_layer = layer; + } + } + + if (!input_layer || !output_layer) { + std::cerr << "Error: Could not find input/output layers" << '\n'; + return 1; + } + + auto total_start_time = std::chrono::high_resolution_clock::now(); + int total_inference_time = 0; + int batch_count = 0; + + for (size_t batch_start = 0; batch_start < num_photo; + batch_start += batch_size) { + size_t batch_end = std::min(batch_start + batch_size, num_photo); + size_t current_batch_size = batch_end - batch_start; + + std::vector batch_data; + batch_data.reserve(current_batch_size * image_size); + + size_t batch_offset = batch_start * image_size; + batch_data.insert(batch_data.end(), all_image_data.begin() + batch_offset, + all_image_data.begin() + batch_offset + + current_batch_size * image_size); + + it_lab_ai::Shape batch_input_shape( + {current_batch_size, static_cast(channels), + static_cast(height), static_cast(width)}); + it_lab_ai::Tensor batch_input = make_tensor(batch_data, batch_input_shape); + + it_lab_ai::Shape batch_output_shape({current_batch_size, output_classes}); + it_lab_ai::Tensor batch_output(batch_output_shape, it_lab_ai::Type::kFloat); + + graph.setInput(input_layer, batch_input); + graph.setOutput(output_layer, batch_output); - bool found_in_top5 = false; - for (int top_k = 0; top_k < std::min(5, static_cast(indices.size())); - ++top_k) { - if (indices[top_k] == static_cast(true_label)) { - found_in_top5 = true; - break; + auto batch_start_time = std::chrono::high_resolution_clock::now(); + graph.inference(options); + auto batch_end_time = std::chrono::high_resolution_clock::now(); + + int batch_time = + static_cast(std::chrono::duration_cast( + batch_end_time - batch_start_time) + .count()); + total_inference_time += batch_time; + batch_count++; + + const std::vector& raw_batch_output = *batch_output.as(); + + for (size_t i = 0; i < current_batch_size; ++i) { + size_t global_idx = batch_start + i; + + std::vector single_output( + raw_batch_output.begin() + i * output_classes, + raw_batch_output.begin() + (i + 1) * output_classes); + + float max_val = + *std::max_element(single_output.begin(), single_output.end()); + float sum = 0.0f; + for (float& val : single_output) { + val = exp(val - max_val); + sum += val; + } + for (float& val : single_output) { + val /= sum; + } + + std::vector indices(single_output.size()); + std::iota(indices.begin(), indices.end(), 0); + std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) { + return single_output[a] > single_output[b]; + }); + + if (indices[0] == static_cast(true_labels[global_idx])) { + correct_predictions_top1++; + } + + for (int top_k = 0; top_k < std::min(5, static_cast(indices.size())); + ++top_k) { + if (indices[top_k] == static_cast(true_labels[global_idx])) { + correct_predictions_top5++; + break; + } } } - if (found_in_top5) { - correct_predictions_top5++; - } + + batch_data.clear(); + batch_data.shrink_to_fit(); } + auto total_end_time = std::chrono::high_resolution_clock::now(); + int total_time = + static_cast(std::chrono::duration_cast( + total_end_time - total_start_time) + .count()); + + std::cout << "\n!INFERENCE TIME INFO START!" << '\n'; + std::cout << "Total inference time (sum of batches): " << total_inference_time + << " ms\n"; + std::cout << "Total wall-clock time for all batches: " << total_time + << " ms\n"; + std::cout << "Number of batches: " << batch_count << '\n'; + std::cout << "Average time per batch: " + << (batch_count > 0 ? total_inference_time / batch_count : 0) + << " ms\n"; + std::cout << "!INFERENCE TIME INFO END!" << '\n'; + double final_accuracy_top1 = - (static_cast(correct_predictions_top1) / total_images) * 100; + (static_cast(correct_predictions_top1) / num_photo) * 100; double final_accuracy_top5 = - (static_cast(correct_predictions_top5) / total_images) * 100; + (static_cast(correct_predictions_top5) / num_photo) * 100; std::cout << "\nFinal Results:" << '\n'; std::cout << "Model: " << model_name << '\n'; std::cout << "Dataset: " << dataset_path << '\n'; - std::cout << "Total images: " << total_images << '\n'; + std::cout << "Total images: " << num_photo << '\n'; + std::cout << "Batch size: " << batch_size << '\n'; std::cout << "Correct predictions (Top-1): " << correct_predictions_top1 << '\n'; std::cout << "Correct predictions (Top-5): " << correct_predictions_top5 diff --git a/app/Graph/build.cpp b/app/Graph/build.cpp index 381e8fcd..cc61ff29 100644 --- a/app/Graph/build.cpp +++ b/app/Graph/build.cpp @@ -1129,19 +1129,19 @@ it_lab_ai::Tensor prepare_mnist_image(const cv::Mat& image) { return it_lab_ai::make_tensor(res, sh); } -void print_time_stats(Graph& graph) { +int print_time_stats(Graph& graph) { #ifdef ENABLE_STATISTIC_TIME std::vector times = graph.getTimeInfo(); - std::cout << "!INFERENCE TIME INFO START!" << '\n'; - for (size_t i = 0; i < times.size(); i++) { + // std::cout << "!INFERENCE TIME INFO START!" << '\n'; + /*for (size_t i = 0; i < times.size(); i++) { std::cout << times[i] << '\n'; - } + }*/ std::vector elps_time = graph.getTime(); int sum = std::accumulate(elps_time.begin(), elps_time.end(), 0); - std::cout << "Elapsed inference time:" << sum << '\n'; - std::cout << "!INFERENCE TIME INFO END!" << '\n'; - graph.printLayerStats(); + // graph.printLayerStats(); + return sum; #else (void)graph; + return 0; #endif } diff --git a/app/Graph/build.hpp b/app/Graph/build.hpp index 5628e9d1..4c5920dd 100644 --- a/app/Graph/build.hpp +++ b/app/Graph/build.hpp @@ -76,7 +76,7 @@ it_lab_ai::Tensor prepare_image(const cv::Mat& image, const std::string& model_name = ""); it_lab_ai::Tensor prepare_mnist_image(const cv::Mat& image); -void print_time_stats(it_lab_ai::Graph& graph); +int print_time_stats(it_lab_ai::Graph& graph); namespace it_lab_ai { class LayerFactory { public: