AutoMM Detection - Quick Start on a Tiny COCO Format Dataset¶
In this section, our goal is to fast finetune a pretrained model on a small dataset in COCO format, and evaluate on its test set. Both training and test sets are in COCO format. See Convert Data to COCO Format for how to convert other datasets to COCO format.
Setting up the imports¶
To start, make sure mmcv and mmdet are installed.
Note: MMDet is no longer actively maintained and is only compatible with MMCV version 2.1.0. Installation can be problematic due to CUDA version compatibility issues. For best results:
Use CUDA 12.4 with PyTorch 2.5
Before installation, run:
pip install -U pip setuptools wheel sudo apt-get install -y ninja-build gcc g++
This will help prevent MMCV installation from hanging during wheel building.
After installation in Jupyter notebook, restart the kernel for changes to take effect.
# Update package tools and install build dependencies
!pip install -U pip setuptools wheel
!sudo apt-get install -y ninja-build gcc g++
# Install MMCV
!python3 -m mim install "mmcv==2.1.0"
# For Google Colab users: If the above fails, use this alternative MMCV installation
# pip install "mmcv==2.1.0" -f https://download.openmmlab.com/mmcv/dist/cu121/torch2.1.0/index.html
# Install MMDet
!python3 -m pip install "mmdet==3.2.0"
# Install MMEngine (version >=0.10.6 for PyTorch 2.5 compatibility)
!python3 -m pip install "mmengine>=0.10.6"
To start, let’s import MultiModalPredictor:
from autogluon.multimodal import MultiModalPredictor
/home/ci/opt/venv/lib/python3.12/site-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.6.3) or chardet (6.0.0.post1)/charset_normalizer (3.4.4) doesn't match a supported version!
warnings.warn(
---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
Cell In[3], line 1
----> 1 from autogluon.multimodal import MultiModalPredictor
File ~/autogluon/multimodal/src/autogluon/multimodal/__init__.py:8
5 except ImportError:
6 pass
----> 8 from .predictor import MultiModalPredictor
10 _add_stream_handler()
File ~/autogluon/multimodal/src/autogluon/multimodal/predictor.py:19
16 from autogluon.core.metrics import Scorer
18 from .constants import AUTOMM_TUTORIAL_MODE, FEW_SHOT_CLASSIFICATION, NER, OBJECT_DETECTION, SEMANTIC_SEGMENTATION
---> 19 from .learners import (
20 BaseLearner,
21 EnsembleLearner,
22 FewShotSVMLearner,
23 MatchingLearner,
24 NERLearner,
25 ObjectDetectionLearner,
26 SemanticSegmentationLearner,
27 )
28 from .utils import get_dir_ckpt_paths
29 from .utils.problem_types import PROBLEM_TYPES_REG
File ~/autogluon/multimodal/src/autogluon/multimodal/learners/__init__.py:1
----> 1 from .base import BaseLearner
2 from .ensemble import EnsembleLearner
3 from .few_shot_svm import FewShotSVMLearner
File ~/autogluon/multimodal/src/autogluon/multimodal/learners/base.py:71
31 from .. import version as ag_version
32 from ..constants import (
33 BEST,
34 BEST_K_MODELS_FILE,
(...)
69 ZERO_SHOT_IMAGE_CLASSIFICATION,
70 )
---> 71 from ..data import (
72 BaseDataModule,
73 MultiModalFeaturePreprocessor,
74 create_fusion_data_processors,
75 data_to_df,
76 get_mixup,
77 infer_column_types,
78 infer_dtypes_by_model_names,
79 infer_output_shape,
80 infer_problem_type,
81 infer_scarcity_mode_by_data_size,
82 init_df_preprocessor,
83 is_image_column,
84 split_train_tuning_data,
85 turn_on_off_feature_column_info,
86 )
87 from ..models import (
88 create_fusion_model,
89 get_model_postprocess_fn,
(...)
92 select_model,
93 )
94 from ..optim import (
95 compute_score,
96 get_aug_loss_func,
(...)
103 infer_metrics,
104 )
File ~/autogluon/multimodal/src/autogluon/multimodal/data/__init__.py:1
----> 1 from .datamodule import BaseDataModule
2 from .dataset import BaseDataset
3 from .dataset_mmlab import MultiImageMixDataset
File ~/autogluon/multimodal/src/autogluon/multimodal/data/datamodule.py:8
5 from torch.utils.data import DataLoader, Dataset
7 from ..constants import PREDICT, TEST, TRAIN, VALIDATE
----> 8 from .dataset import BaseDataset
9 from .preprocess_dataframe import MultiModalFeaturePreprocessor
10 from .utils import get_collate_fn
File ~/autogluon/multimodal/src/autogluon/multimodal/data/dataset.py:9
7 from ..constants import GET_ITEM_ERROR_RETRY
8 from .preprocess_dataframe import MultiModalFeaturePreprocessor
----> 9 from .utils import apply_data_processor, apply_df_preprocessor, get_per_sample_features
11 logger = logging.getLogger(__name__)
14 class BaseDataset(torch.utils.data.Dataset):
File ~/autogluon/multimodal/src/autogluon/multimodal/data/utils.py:48
46 from .process_numerical import NumericalProcessor
47 from .process_semantic_seg_img import SemanticSegImageProcessor
---> 48 from .process_text import TextProcessor
50 logger = logging.getLogger(__name__)
53 def get_collate_fn(
54 df_preprocessor: Union[MultiModalFeaturePreprocessor, List[MultiModalFeaturePreprocessor]],
55 data_processors: Union[Dict, List[Dict]],
56 per_gpu_batch_size: Optional[int] = None,
57 ):
File ~/autogluon/multimodal/src/autogluon/multimodal/data/process_text.py:19
17 from ..models.utils import get_pretrained_tokenizer
18 from .collator import PadCollator, StackCollator
---> 19 from .template_engine import TemplateEngine
20 from .trivial_augmenter import TrivialAugment
22 logger = logging.getLogger(__name__)
File ~/autogluon/multimodal/src/autogluon/multimodal/data/template_engine.py:6
3 import numpy as np
4 from omegaconf import DictConfig, OmegaConf
----> 6 from .templates import DatasetTemplates, Template, TemplateCollection
8 logger = logging.getLogger(__name__)
11 class TemplateEngine:
File ~/autogluon/multimodal/src/autogluon/multimodal/data/templates.py:16
13 from typing import Dict, List, Optional, Tuple
15 import pandas as pd
---> 16 import pkg_resources
17 import yaml
18 from jinja2 import BaseLoader, Environment, meta
ModuleNotFoundError: No module named 'pkg_resources'
And also import some other packages that will be used in this tutorial:
import os
import time
from autogluon.core.utils.loaders import load_zip
Downloading Data¶
We have the sample dataset ready in the cloud. Let’s download it:
zip_file = "https://automl-mm-bench.s3.amazonaws.com/object_detection_dataset/tiny_motorbike_coco.zip"
download_dir = "./tiny_motorbike_coco"
load_zip.unzip(zip_file, unzip_dir=download_dir)
data_dir = os.path.join(download_dir, "tiny_motorbike")
train_path = os.path.join(data_dir, "Annotations", "trainval_cocoformat.json")
test_path = os.path.join(data_dir, "Annotations", "test_cocoformat.json")
Dataset Format¶
For COCO format datasets, provide JSON annotation files for each split:
trainval_cocoformat.json: train and validation datatest_cocoformat.json: test data
Model Selection¶
We use the medium_quality preset which features:
Base model: YOLOX-large (pretrained on COCO)
Benefits: Fast finetuning, quick inference, easy deployment
Alternative presets available:
high_quality: DINO-Resnet50 modelbest_quality: DINO-SwinL model
Both alternatives offer improved performance at the cost of slower processing and higher GPU memory requirements.
presets = "medium_quality"
When creating the MultiModalPredictor, specify these essential parameters:
problem_type="object_detection"to define the taskpresets="medium_quality"for presets selectionsample_data_pathpointing to any dataset split (typically train_path) to infer object categoriespath(optional) to set a custom save location
If no path is specified, the model will be automatically saved to a timestamped directory under AutogluonModels/.
# Init predictor
import uuid
model_path = f"./tmp/{uuid.uuid4().hex}-quick_start_tutorial_temp_save"
predictor = MultiModalPredictor(
problem_type="object_detection",
sample_data_path=train_path,
presets=presets,
path=model_path,
)
Finetuning the Model¶
The model uses optimized preset configurations for learning rate, epochs, and batch size. By default, it employs a two-stage learning rate strategy:
Model head layers use 100x higher learning rate This approach accelerates convergence and typically improves performance, especially for small datasets (hundreds to thousands of images)
Timing results below are from a test run on AWS g4.2xlarge EC2 instance:
start = time.time()
predictor.fit(train_path) # Fit
train_end = time.time()
Notice that at the end of each progress bar, if the checkpoint at current stage is saved,
it prints the model’s save path.
In this example, it’s ./quick_start_tutorial_temp_save.
Print out the time and we can see that it’s fast!
print("This finetuning takes %.2f seconds." % (train_end - start))
Evaluation¶
To evaluate the model we just trained, run following code.
And the evaluation results are shown in command line output. The first line is mAP in COCO standard, and the second line is mAP in VOC standard (or mAP50). For more details about these metrics, see COCO’s evaluation guideline. Note that for presenting a fast finetuning we use presets “medium_quality”, you could get better result on this dataset by simply using “high_quality” or “best_quality” presets, or customize your own model and hyperparameter settings: Customization, and some other examples at Fast Fine-tune Coco or High Performance Fine-tune Coco.
predictor.evaluate(test_path)
eval_end = time.time()
Print out the evaluation time:
print("The evaluation takes %.2f seconds." % (eval_end - train_end))
We can load a new predictor with previous save path, and we can also reset the number of used GPUs if not all the devices are available:
# Load and reset num_gpus
new_predictor = MultiModalPredictor.load(model_path)
new_predictor.set_num_gpus(1)
Evaluating the new predictor gives us exactly the same result:
# Evaluate new predictor
new_predictor.evaluate(test_path)
For how to set the hyperparameters and finetune the model with higher performance, see AutoMM Detection - High Performance Finetune on COCO Format Dataset.
Inference¶
Let’s perform predictions using our finetuned model. The predictor can process the entire test set with a single command:
pred = predictor.predict(test_path)
print(len(pred)) # Number of predictions
print(pred[:3]) # Sample of first 3 predictions
The predictor returns predictions as a pandas DataFrame with two columns:
image: Contains path to each input imagebboxes: Contains list of detected objects, where each object is a dictionary:{ "class": "predicted_class_name", "bbox": [x1, y1, x2, y2], # Coordinates of Upper Left and Bottom Right corners "score": confidence_score }
By default, predictions are returned but not saved. To save detection results, use the save parameter in your predict call.
# To save as csv format
pred = predictor.predict(test_path, save_results=True, as_coco=False)
# Or to save as COCO format. Note that the `pred` returned is always a pandas dataframe.
pred = predictor.predict(test_path, save_results=True, as_coco=True, result_save_path="./results.json")
The predictions can be saved in two formats:
CSV file: Matches the DataFrame structure with image and bboxes columns
COCO JSON: Standard COCO format annotation file
This works with any predictor configuration (pretrained or finetuned models).
Visualizing Results¶
To run visualizations, ensure that you have opencv installed. If you haven’t already, install opencv by running
!pip install opencv-python
To visualize the detection bounding boxes, run the following:
from autogluon.multimodal.utils import ObjectDetectionVisualizer
conf_threshold = 0.4 # Specify a confidence threshold to filter out unwanted boxes
image_result = pred.iloc[30]
img_path = image_result.image # Select an image to visualize
visualizer = ObjectDetectionVisualizer(img_path) # Initialize the Visualizer
out = visualizer.draw_instance_predictions(image_result, conf_threshold=conf_threshold) # Draw detections
visualized = out.get_image() # Get the visualized image
from PIL import Image
from IPython.display import display
img = Image.fromarray(visualized, 'RGB')
display(img)
Testing on Your Own Data¶
You can also predict on your own images with various input format. The follow is an example:
Download the example image:
from autogluon.multimodal.utils import download
image_url = "https://raw.githubusercontent.com/dmlc/web-data/master/gluoncv/detection/street_small.jpg"
test_image = download(image_url)
Run inference on data in a json file of COCO format (See Convert Data to COCO Format for more details about COCO format). Note that since the root is by default the parent folder of the annotation file, here we put the annotation file in a folder:
import json
# create a input file for demo
data = {"images": [{"id": 0, "width": -1, "height": -1, "file_name": test_image}], "categories": []}
os.mkdir("input_data_for_demo")
input_file = "input_data_for_demo/demo_annotation.json"
with open(input_file, "w+") as f:
json.dump(data, f)
pred_test_image = predictor.predict(input_file)
print(pred_test_image)
Run inference on data in a list of image file names:
pred_test_image = predictor.predict([test_image])
print(pred_test_image)
Other Examples¶
You may go to AutoMM Examples to explore other examples about AutoMM.
Customization¶
To learn how to customize AutoMM, please refer to Customize AutoMM.
Citation¶
@article{DBLP:journals/corr/abs-2107-08430,
author = {Zheng Ge and
Songtao Liu and
Feng Wang and
Zeming Li and
Jian Sun},
title = {{YOLOX:} Exceeding {YOLO} Series in 2021},
journal = {CoRR},
volume = {abs/2107.08430},
year = {2021},
url = {https://arxiv.org/abs/2107.08430},
eprinttype = {arXiv},
eprint = {2107.08430},
timestamp = {Tue, 05 Apr 2022 14:09:44 +0200},
biburl = {https://dblp.org/rec/journals/corr/abs-2107-08430.bib},
bibsource = {dblp computer science bibliography, https://dblp.org},
}