Kye
1 year ago
parent
1fb272d791
commit
78d6a164fc
|
Before Width: | Height: | Size: 256 KiB |
|
Before Width: | Height: | Size: 1.2 MiB |
|
Before Width: | Height: | Size: 1.1 MiB |
|
Before Width: | Height: | Size: 286 KiB |
|
Before Width: | Height: | Size: 555 KiB |
|
Before Width: | Height: | Size: 120 KiB |
|
Before Width: | Height: | Size: 373 KiB |
|
Before Width: | Height: | Size: 354 KiB |
|
Before Width: | Height: | Size: 2.8 MiB |
|
Before Width: | Height: | Size: 472 KiB |
|
Before Width: | Height: | Size: 456 KiB |
@ -1,146 +0,0 @@
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# vscode
|
||||
.vscode/
|
||||
output/
|
||||
outputs/
|
||||
subs/
|
||||
logs/
|
||||
|
||||
grounding/config/configs
|
||||
grounding/version.py
|
||||
|
||||
vis/
|
||||
tmp/
|
||||
@ -1,201 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2023 - present, IDEA Research.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@ -1,327 +0,0 @@
|
||||
<div align="center">
|
||||
<img src="./.asset/grounding_dino_logo.png" width="30%">
|
||||
</div>
|
||||
|
||||
# :sauropod: Grounding DINO
|
||||
|
||||
[](https://paperswithcode.com/sota/zero-shot-object-detection-on-mscoco?p=grounding-dino-marrying-dino-with-grounded) [](https://paperswithcode.com/sota/zero-shot-object-detection-on-odinw?p=grounding-dino-marrying-dino-with-grounded) \
|
||||
[](https://paperswithcode.com/sota/object-detection-on-coco-minival?p=grounding-dino-marrying-dino-with-grounded) [](https://paperswithcode.com/sota/object-detection-on-coco?p=grounding-dino-marrying-dino-with-grounded)
|
||||
|
||||
|
||||
**[IDEA-CVR, IDEA-Research](https://github.com/IDEA-Research)**
|
||||
|
||||
[Shilong Liu](http://www.lsl.zone/), [Zhaoyang Zeng](https://scholar.google.com/citations?user=U_cvvUwAAAAJ&hl=zh-CN&oi=ao), [Tianhe Ren](https://rentainhe.github.io/), [Feng Li](https://scholar.google.com/citations?user=ybRe9GcAAAAJ&hl=zh-CN), [Hao Zhang](https://scholar.google.com/citations?user=B8hPxMQAAAAJ&hl=zh-CN), [Jie Yang](https://github.com/yangjie-cv), [Chunyuan Li](https://scholar.google.com/citations?user=Zd7WmXUAAAAJ&hl=zh-CN&oi=ao), [Jianwei Yang](https://jwyang.github.io/), [Hang Su](https://scholar.google.com/citations?hl=en&user=dxN1_X0AAAAJ&view_op=list_works&sortby=pubdate), [Jun Zhu](https://scholar.google.com/citations?hl=en&user=axsP38wAAAAJ), [Lei Zhang](https://www.leizhang.org/)<sup>:email:</sup>.
|
||||
|
||||
|
||||
[[`Paper`](https://arxiv.org/abs/2303.05499)] [[`Demo`](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo)] [[`BibTex`](#black_nib-citation)]
|
||||
|
||||
|
||||
PyTorch implementation and pretrained models for Grounding DINO. For details, see the paper **[Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection](https://arxiv.org/abs/2303.05499)**.
|
||||
|
||||
## :sun_with_face: Helpful Tutorial
|
||||
|
||||
- :grapes: [[Read our arXiv Paper](https://arxiv.org/abs/2303.05499)]
|
||||
- :apple: [[Watch our simple introduction video on YouTube](https://youtu.be/wxWDt5UiwY8)]
|
||||
- :blossom: [[Try the Colab Demo](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/zero-shot-object-detection-with-grounding-dino.ipynb)]
|
||||
- :sunflower: [[Try our Official Huggingface Demo](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo)]
|
||||
- :maple_leaf: [[Watch the Step by Step Tutorial about GroundingDINO by Roboflow AI](https://youtu.be/cMa77r3YrDk)]
|
||||
- :mushroom: [[GroundingDINO: Automated Dataset Annotation and Evaluation by Roboflow AI](https://youtu.be/C4NqaRBz_Kw)]
|
||||
- :hibiscus: [[Accelerate Image Annotation with SAM and GroundingDINO by Roboflow AI](https://youtu.be/oEQYStnF2l8)]
|
||||
- :white_flower: [[Autodistill: Train YOLOv8 with ZERO Annotations based on Grounding-DINO and Grounded-SAM by Roboflow AI](https://github.com/autodistill/autodistill)]
|
||||
|
||||
<!-- Grounding DINO Methods |
|
||||
[](https://arxiv.org/abs/2303.05499)
|
||||
[](https://youtu.be/wxWDt5UiwY8) -->
|
||||
|
||||
<!-- Grounding DINO Demos |
|
||||
[](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/zero-shot-object-detection-with-grounding-dino.ipynb) -->
|
||||
<!-- [](https://youtu.be/cMa77r3YrDk)
|
||||
[](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo)
|
||||
[](https://youtu.be/oEQYStnF2l8)
|
||||
[](https://youtu.be/C4NqaRBz_Kw) -->
|
||||
|
||||
## :sparkles: Highlight Projects
|
||||
|
||||
- [DetGPT: Detect What You Need via Reasoning](https://github.com/OptimalScale/DetGPT)
|
||||
- [Grounded-SAM: Marrying Grounding DINO with Segment Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything)
|
||||
- [Grounding DINO with Stable Diffusion](demo/image_editing_with_groundingdino_stablediffusion.ipynb)
|
||||
- [Grounding DINO with GLIGEN for Controllable Image Editing](demo/image_editing_with_groundingdino_gligen.ipynb)
|
||||
- [OpenSeeD: A Simple and Strong Openset Segmentation Model](https://github.com/IDEA-Research/OpenSeeD)
|
||||
- [SEEM: Segment Everything Everywhere All at Once](https://github.com/UX-Decoder/Segment-Everything-Everywhere-All-At-Once)
|
||||
- [X-GPT: Conversational Visual Agent supported by X-Decoder](https://github.com/microsoft/X-Decoder/tree/xgpt)
|
||||
- [GLIGEN: Open-Set Grounded Text-to-Image Generation](https://github.com/gligen/GLIGEN)
|
||||
- [LLaVA: Large Language and Vision Assistant](https://github.com/haotian-liu/LLaVA)
|
||||
|
||||
<!-- Extensions | [Grounding DINO with Segment Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything); [Grounding DINO with Stable Diffusion](demo/image_editing_with_groundingdino_stablediffusion.ipynb); [Grounding DINO with GLIGEN](demo/image_editing_with_groundingdino_gligen.ipynb) -->
|
||||
|
||||
|
||||
|
||||
<!-- Official PyTorch implementation of [Grounding DINO](https://arxiv.org/abs/2303.05499), a stronger open-set object detector. Code is available now! -->
|
||||
|
||||
|
||||
## :bulb: Highlight
|
||||
|
||||
- **Open-Set Detection.** Detect **everything** with language!
|
||||
- **High Performancce.** COCO zero-shot **52.5 AP** (training without COCO data!). COCO fine-tune **63.0 AP**.
|
||||
- **Flexible.** Collaboration with Stable Diffusion for Image Editting.
|
||||
|
||||
|
||||
|
||||
|
||||
## :fire: News
|
||||
- **`2023/06/17`**: We provide an example to evaluate Grounding DINO on COCO zero-shot performance.
|
||||
- **`2023/04/15`**: Refer to [CV in the Wild Readings](https://github.com/Computer-Vision-in-the-Wild/CVinW_Readings) for those who are interested in open-set recognition!
|
||||
- **`2023/04/08`**: We release [demos](demo/image_editing_with_groundingdino_gligen.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [GLIGEN](https://github.com/gligen/GLIGEN) for more controllable image editings.
|
||||
- **`2023/04/08`**: We release [demos](demo/image_editing_with_groundingdino_stablediffusion.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [Stable Diffusion](https://github.com/Stability-AI/StableDiffusion) for image editings.
|
||||
- **`2023/04/06`**: We build a new demo by marrying GroundingDINO with [Segment-Anything](https://github.com/facebookresearch/segment-anything) named **[Grounded-Segment-Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything)** aims to support segmentation in GroundingDINO.
|
||||
- **`2023/03/28`**: A YouTube [video](https://youtu.be/cMa77r3YrDk) about Grounding DINO and basic object detection prompt engineering. [[SkalskiP](https://github.com/SkalskiP)]
|
||||
- **`2023/03/28`**: Add a [demo](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo) on Hugging Face Space!
|
||||
- **`2023/03/27`**: Support CPU-only mode. Now the model can run on machines without GPUs.
|
||||
- **`2023/03/25`**: A [demo](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/zero-shot-object-detection-with-grounding-dino.ipynb) for Grounding DINO is available at Colab. [[SkalskiP](https://github.com/SkalskiP)]
|
||||
- **`2023/03/22`**: Code is available Now!
|
||||
|
||||
<details open>
|
||||
<summary><font size="4">
|
||||
Description
|
||||
</font></summary>
|
||||
<a href="https://arxiv.org/abs/2303.05499">Paper</a> introduction.
|
||||
<img src=".asset/hero_figure.png" alt="ODinW" width="100%">
|
||||
Marrying <a href="https://github.com/IDEA-Research/GroundingDINO">Grounding DINO</a> and <a href="https://github.com/gligen/GLIGEN">GLIGEN</a>
|
||||
<img src="https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/GD_GLIGEN.png" alt="gd_gligen" width="100%">
|
||||
</details>
|
||||
|
||||
## :star: Explanations/Tips for Grounding DINO Inputs and Outputs
|
||||
- Grounding DINO accepts an `(image, text)` pair as inputs.
|
||||
- It outputs `900` (by default) object boxes. Each box has similarity scores across all input words. (as shown in Figures below.)
|
||||
- We defaultly choose the boxes whose highest similarities are higher than a `box_threshold`.
|
||||
- We extract the words whose similarities are higher than the `text_threshold` as predicted labels.
|
||||
- If you want to obtain objects of specific phrases, like the `dogs` in the sentence `two dogs with a stick.`, you can select the boxes with highest text similarities with `dogs` as final outputs.
|
||||
- Note that each word can be split to **more than one** tokens with different tokenlizers. The number of words in a sentence may not equal to the number of text tokens.
|
||||
- We suggest separating different category names with `.` for Grounding DINO.
|
||||
|
||||
|
||||
|
||||
## :label: TODO
|
||||
|
||||
- [x] Release inference code and demo.
|
||||
- [x] Release checkpoints.
|
||||
- [x] Grounding DINO with Stable Diffusion and GLIGEN demos.
|
||||
- [ ] Release training codes.
|
||||
|
||||
## :hammer_and_wrench: Install
|
||||
|
||||
**Note:**
|
||||
|
||||
If you have a CUDA environment, please make sure the environment variable `CUDA_HOME` is set. It will be compiled under CPU-only mode if no CUDA available.
|
||||
|
||||
**Installation:**
|
||||
|
||||
Clone the GroundingDINO repository from GitHub.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/IDEA-Research/GroundingDINO.git
|
||||
```
|
||||
|
||||
Change the current directory to the GroundingDINO folder.
|
||||
|
||||
```bash
|
||||
cd GroundingDINO/
|
||||
```
|
||||
|
||||
Install the required dependencies in the current directory.
|
||||
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
Download pre-trained model weights.
|
||||
|
||||
```bash
|
||||
mkdir weights
|
||||
cd weights
|
||||
wget -q https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth
|
||||
cd ..
|
||||
```
|
||||
|
||||
## :arrow_forward: Demo
|
||||
Check your GPU ID (only if you're using a GPU)
|
||||
|
||||
```bash
|
||||
nvidia-smi
|
||||
```
|
||||
Replace `{GPU ID}`, `image_you_want_to_detect.jpg`, and `"dir you want to save the output"` with appropriate values in the following command
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES={GPU ID} python demo/inference_on_a_image.py \
|
||||
-c groundingdino/config/GroundingDINO_SwinT_OGC.py \
|
||||
-p weights/groundingdino_swint_ogc.pth \
|
||||
-i image_you_want_to_detect.jpg \
|
||||
-o "dir you want to save the output" \
|
||||
-t "chair"
|
||||
[--cpu-only] # open it for cpu mode
|
||||
```
|
||||
|
||||
If you would like to specify the phrases to detect, here is a demo:
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES={GPU ID} python demo/inference_on_a_image.py \
|
||||
-c groundingdino/config/GroundingDINO_SwinT_OGC.py \
|
||||
-p ./groundingdino_swint_ogc.pth \
|
||||
-i .asset/cat_dog.jpeg \
|
||||
-o logs/1111 \
|
||||
-t "There is a cat and a dog in the image ." \
|
||||
--token_spans "[[[9, 10], [11, 14]], [[19, 20], [21, 24]]]"
|
||||
[--cpu-only] # open it for cpu mode
|
||||
```
|
||||
The token_spans specify the start and end positions of a phrases. For example, the first phrase is `[[9, 10], [11, 14]]`. `"There is a cat and a dog in the image ."[9:10] = 'a'`, `"There is a cat and a dog in the image ."[11:14] = 'cat'`. Hence it refers to the phrase `a cat` . Similarly, the `[[19, 20], [21, 24]]` refers to the phrase `a dog`.
|
||||
|
||||
See the `demo/inference_on_a_image.py` for more details.
|
||||
|
||||
**Running with Python:**
|
||||
|
||||
```python
|
||||
from groundingdino.util.inference import load_model, load_image, predict, annotate
|
||||
import cv2
|
||||
|
||||
model = load_model("groundingdino/config/GroundingDINO_SwinT_OGC.py", "weights/groundingdino_swint_ogc.pth")
|
||||
IMAGE_PATH = "weights/dog-3.jpeg"
|
||||
TEXT_PROMPT = "chair . person . dog ."
|
||||
BOX_TRESHOLD = 0.35
|
||||
TEXT_TRESHOLD = 0.25
|
||||
|
||||
image_source, image = load_image(IMAGE_PATH)
|
||||
|
||||
boxes, logits, phrases = predict(
|
||||
model=model,
|
||||
image=image,
|
||||
caption=TEXT_PROMPT,
|
||||
box_threshold=BOX_TRESHOLD,
|
||||
text_threshold=TEXT_TRESHOLD
|
||||
)
|
||||
|
||||
annotated_frame = annotate(image_source=image_source, boxes=boxes, logits=logits, phrases=phrases)
|
||||
cv2.imwrite("annotated_image.jpg", annotated_frame)
|
||||
```
|
||||
**Web UI**
|
||||
|
||||
We also provide a demo code to integrate Grounding DINO with Gradio Web UI. See the file `demo/gradio_app.py` for more details.
|
||||
|
||||
**Notebooks**
|
||||
|
||||
- We release [demos](demo/image_editing_with_groundingdino_gligen.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [GLIGEN](https://github.com/gligen/GLIGEN) for more controllable image editings.
|
||||
- We release [demos](demo/image_editing_with_groundingdino_stablediffusion.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [Stable Diffusion](https://github.com/Stability-AI/StableDiffusion) for image editings.
|
||||
|
||||
## COCO Zero-shot Evaluations
|
||||
|
||||
We provide an example to evaluate Grounding DINO zero-shot performance on COCO. The results should be **48.5**.
|
||||
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES=0 \
|
||||
python demo/test_ap_on_coco.py \
|
||||
-c groundingdino/config/GroundingDINO_SwinT_OGC.py \
|
||||
-p weights/groundingdino_swint_ogc.pth \
|
||||
--anno_path /path/to/annoataions/ie/instances_val2017.json \
|
||||
--image_dir /path/to/imagedir/ie/val2017
|
||||
```
|
||||
|
||||
|
||||
## :luggage: Checkpoints
|
||||
|
||||
<!-- insert a table -->
|
||||
<table>
|
||||
<thead>
|
||||
<tr style="text-align: right;">
|
||||
<th></th>
|
||||
<th>name</th>
|
||||
<th>backbone</th>
|
||||
<th>Data</th>
|
||||
<th>box AP on COCO</th>
|
||||
<th>Checkpoint</th>
|
||||
<th>Config</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<th>1</th>
|
||||
<td>GroundingDINO-T</td>
|
||||
<td>Swin-T</td>
|
||||
<td>O365,GoldG,Cap4M</td>
|
||||
<td>48.4 (zero-shot) / 57.2 (fine-tune)</td>
|
||||
<td><a href="https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth">GitHub link</a> | <a href="https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/groundingdino_swint_ogc.pth">HF link</a></td>
|
||||
<td><a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/groundingdino/config/GroundingDINO_SwinT_OGC.py">link</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>2</th>
|
||||
<td>GroundingDINO-B</td>
|
||||
<td>Swin-B</td>
|
||||
<td>COCO,O365,GoldG,Cap4M,OpenImage,ODinW-35,RefCOCO</td>
|
||||
<td>56.7 </td>
|
||||
<td><a href="https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha2/groundingdino_swinb_cogcoor.pth">GitHub link</a> | <a href="https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/groundingdino_swinb_cogcoor.pth">HF link</a>
|
||||
<td><a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/groundingdino/config/GroundingDINO_SwinB.cfg.py">link</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
## :medal_military: Results
|
||||
|
||||
<details open>
|
||||
<summary><font size="4">
|
||||
COCO Object Detection Results
|
||||
</font></summary>
|
||||
<img src=".asset/COCO.png" alt="COCO" width="100%">
|
||||
</details>
|
||||
|
||||
<details open>
|
||||
<summary><font size="4">
|
||||
ODinW Object Detection Results
|
||||
</font></summary>
|
||||
<img src=".asset/ODinW.png" alt="ODinW" width="100%">
|
||||
</details>
|
||||
|
||||
<details open>
|
||||
<summary><font size="4">
|
||||
Marrying Grounding DINO with <a href="https://github.com/Stability-AI/StableDiffusion">Stable Diffusion</a> for Image Editing
|
||||
</font></summary>
|
||||
See our example <a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/demo/image_editing_with_groundingdino_stablediffusion.ipynb">notebook</a> for more details.
|
||||
<img src=".asset/GD_SD.png" alt="GD_SD" width="100%">
|
||||
</details>
|
||||
|
||||
|
||||
<details open>
|
||||
<summary><font size="4">
|
||||
Marrying Grounding DINO with <a href="https://github.com/gligen/GLIGEN">GLIGEN</a> for more Detailed Image Editing.
|
||||
</font></summary>
|
||||
See our example <a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/demo/image_editing_with_groundingdino_gligen.ipynb">notebook</a> for more details.
|
||||
<img src=".asset/GD_GLIGEN.png" alt="GD_GLIGEN" width="100%">
|
||||
</details>
|
||||
|
||||
## :sauropod: Model: Grounding DINO
|
||||
|
||||
Includes: a text backbone, an image backbone, a feature enhancer, a language-guided query selection, and a cross-modality decoder.
|
||||
|
||||
|
||||
|
||||
|
||||
## :hearts: Acknowledgement
|
||||
|
||||
Our model is related to [DINO](https://github.com/IDEA-Research/DINO) and [GLIP](https://github.com/microsoft/GLIP). Thanks for their great work!
|
||||
|
||||
We also thank great previous work including DETR, Deformable DETR, SMCA, Conditional DETR, Anchor DETR, Dynamic DETR, DAB-DETR, DN-DETR, etc. More related work are available at [Awesome Detection Transformer](https://github.com/IDEACVR/awesome-detection-transformer). A new toolbox [detrex](https://github.com/IDEA-Research/detrex) is available as well.
|
||||
|
||||
Thanks [Stable Diffusion](https://github.com/Stability-AI/StableDiffusion) and [GLIGEN](https://github.com/gligen/GLIGEN) for their awesome models.
|
||||
|
||||
|
||||
## :black_nib: Citation
|
||||
|
||||
If you find our work helpful for your research, please consider citing the following BibTeX entry.
|
||||
|
||||
```bibtex
|
||||
@article{liu2023grounding,
|
||||
title={Grounding dino: Marrying dino with grounded pre-training for open-set object detection},
|
||||
author={Liu, Shilong and Zeng, Zhaoyang and Ren, Tianhe and Li, Feng and Zhang, Hao and Yang, Jie and Li, Chunyuan and Yang, Jianwei and Su, Hang and Zhu, Jun and others},
|
||||
journal={arXiv preprint arXiv:2303.05499},
|
||||
year={2023}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,120 +0,0 @@
|
||||
import argparse
|
||||
import cv2
|
||||
import os
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
|
||||
|
||||
import warnings
|
||||
|
||||
import torch
|
||||
|
||||
# prepare the environment
|
||||
os.system("python setup.py build develop --user")
|
||||
os.system("pip install packaging==21.3")
|
||||
os.system("pip install gradio")
|
||||
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
import gradio as gr
|
||||
|
||||
from groundingdino.models import build_model
|
||||
from groundingdino.util.slconfig import SLConfig
|
||||
from groundingdino.util.utils import clean_state_dict
|
||||
from groundingdino.util.inference import annotate, predict
|
||||
import groundingdino.datasets.transforms as T
|
||||
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
|
||||
|
||||
# Use this command for evaluate the Grounding DINO model
|
||||
config_file = "groundingdino/config/GroundingDINO_SwinT_OGC.py"
|
||||
ckpt_repo_id = "ShilongLiu/GroundingDINO"
|
||||
ckpt_filenmae = "groundingdino_swint_ogc.pth"
|
||||
|
||||
|
||||
def load_model_hf(model_config_path, repo_id, filename, device='cpu'):
|
||||
args = SLConfig.fromfile(model_config_path)
|
||||
model = build_model(args)
|
||||
args.device = device
|
||||
|
||||
cache_file = hf_hub_download(repo_id=repo_id, filename=filename)
|
||||
checkpoint = torch.load(cache_file, map_location='cpu')
|
||||
log = model.load_state_dict(clean_state_dict(checkpoint['model']), strict=False)
|
||||
print("Model loaded from {} \n => {}".format(cache_file, log))
|
||||
_ = model.eval()
|
||||
return model
|
||||
|
||||
def image_transform_grounding(init_image):
|
||||
transform = T.Compose([
|
||||
T.RandomResize([800], max_size=1333),
|
||||
T.ToTensor(),
|
||||
T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
|
||||
])
|
||||
image, _ = transform(init_image, None) # 3, h, w
|
||||
return init_image, image
|
||||
|
||||
def image_transform_grounding_for_vis(init_image):
|
||||
transform = T.Compose([
|
||||
T.RandomResize([800], max_size=1333),
|
||||
])
|
||||
image, _ = transform(init_image, None) # 3, h, w
|
||||
return image
|
||||
|
||||
model = load_model_hf(config_file, ckpt_repo_id, ckpt_filenmae)
|
||||
|
||||
def run_grounding(input_image, grounding_caption, box_threshold, text_threshold):
|
||||
init_image = input_image.convert("RGB")
|
||||
|
||||
_, image_tensor = image_transform_grounding(init_image)
|
||||
image_pil: Image = image_transform_grounding_for_vis(init_image)
|
||||
|
||||
# run grounidng
|
||||
boxes, logits, phrases = predict(model, image_tensor, grounding_caption, box_threshold, text_threshold, device='cpu')
|
||||
annotated_frame = annotate(image_source=np.asarray(image_pil), boxes=boxes, logits=logits, phrases=phrases)
|
||||
image_with_box = Image.fromarray(cv2.cvtColor(annotated_frame, cv2.COLOR_BGR2RGB))
|
||||
|
||||
|
||||
return image_with_box
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
parser = argparse.ArgumentParser("Grounding DINO demo", add_help=True)
|
||||
parser.add_argument("--debug", action="store_true", help="using debug mode")
|
||||
parser.add_argument("--share", action="store_true", help="share the app")
|
||||
args = parser.parse_args()
|
||||
|
||||
block = gr.Blocks().queue()
|
||||
with block:
|
||||
gr.Markdown("# [Grounding DINO](https://github.com/IDEA-Research/GroundingDINO)")
|
||||
gr.Markdown("### Open-World Detection with Grounding DINO")
|
||||
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
input_image = gr.Image(source='upload', type="pil")
|
||||
grounding_caption = gr.Textbox(label="Detection Prompt")
|
||||
run_button = gr.Button(label="Run")
|
||||
with gr.Accordion("Advanced options", open=False):
|
||||
box_threshold = gr.Slider(
|
||||
label="Box Threshold", minimum=0.0, maximum=1.0, value=0.25, step=0.001
|
||||
)
|
||||
text_threshold = gr.Slider(
|
||||
label="Text Threshold", minimum=0.0, maximum=1.0, value=0.25, step=0.001
|
||||
)
|
||||
|
||||
with gr.Column():
|
||||
gallery = gr.outputs.Image(
|
||||
type="pil",
|
||||
# label="grounding results"
|
||||
).style(full_width=True, full_height=True)
|
||||
# gallery = gr.Gallery(label="Generated images", show_label=False).style(
|
||||
# grid=[1], height="auto", container=True, full_width=True, full_height=True)
|
||||
|
||||
run_button.click(fn=run_grounding, inputs=[
|
||||
input_image, grounding_caption, box_threshold, text_threshold], outputs=[gallery])
|
||||
|
||||
|
||||
block.launch(server_name='0.0.0.0', server_port=7579, debug=args.debug, share=args.share)
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -1,212 +0,0 @@
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
import groundingdino.datasets.transforms as T
|
||||
from groundingdino.models import build_model
|
||||
from groundingdino.util.slconfig import SLConfig
|
||||
from groundingdino.util.utils import clean_state_dict, get_phrases_from_posmap
|
||||
from groundingdino.util.vl_utils import create_positive_map_from_span
|
||||
|
||||
|
||||
def plot_boxes_to_image(image_pil, tgt):
|
||||
H, W = tgt["size"]
|
||||
boxes = tgt["boxes"]
|
||||
labels = tgt["labels"]
|
||||
assert len(boxes) == len(labels), "boxes and labels must have same length"
|
||||
|
||||
draw = ImageDraw.Draw(image_pil)
|
||||
mask = Image.new("L", image_pil.size, 0)
|
||||
mask_draw = ImageDraw.Draw(mask)
|
||||
|
||||
# draw boxes and masks
|
||||
for box, label in zip(boxes, labels):
|
||||
# from 0..1 to 0..W, 0..H
|
||||
box = box * torch.Tensor([W, H, W, H])
|
||||
# from xywh to xyxy
|
||||
box[:2] -= box[2:] / 2
|
||||
box[2:] += box[:2]
|
||||
# random color
|
||||
color = tuple(np.random.randint(0, 255, size=3).tolist())
|
||||
# draw
|
||||
x0, y0, x1, y1 = box
|
||||
x0, y0, x1, y1 = int(x0), int(y0), int(x1), int(y1)
|
||||
|
||||
draw.rectangle([x0, y0, x1, y1], outline=color, width=6)
|
||||
# draw.text((x0, y0), str(label), fill=color)
|
||||
|
||||
font = ImageFont.load_default()
|
||||
if hasattr(font, "getbbox"):
|
||||
bbox = draw.textbbox((x0, y0), str(label), font)
|
||||
else:
|
||||
w, h = draw.textsize(str(label), font)
|
||||
bbox = (x0, y0, w + x0, y0 + h)
|
||||
# bbox = draw.textbbox((x0, y0), str(label))
|
||||
draw.rectangle(bbox, fill=color)
|
||||
draw.text((x0, y0), str(label), fill="white")
|
||||
|
||||
mask_draw.rectangle([x0, y0, x1, y1], fill=255, width=6)
|
||||
|
||||
return image_pil, mask
|
||||
|
||||
|
||||
def load_image(image_path):
|
||||
# load image
|
||||
image_pil = Image.open(image_path).convert("RGB") # load image
|
||||
|
||||
transform = T.Compose(
|
||||
[
|
||||
T.RandomResize([800], max_size=1333),
|
||||
T.ToTensor(),
|
||||
T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
|
||||
]
|
||||
)
|
||||
image, _ = transform(image_pil, None) # 3, h, w
|
||||
return image_pil, image
|
||||
|
||||
|
||||
def load_model(model_config_path, model_checkpoint_path, cpu_only=False):
|
||||
args = SLConfig.fromfile(model_config_path)
|
||||
args.device = "cuda" if not cpu_only else "cpu"
|
||||
model = build_model(args)
|
||||
checkpoint = torch.load(model_checkpoint_path, map_location="cpu")
|
||||
load_res = model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False)
|
||||
print(load_res)
|
||||
_ = model.eval()
|
||||
return model
|
||||
|
||||
|
||||
def get_grounding_output(model, image, caption, box_threshold, text_threshold=None, with_logits=True, cpu_only=False, token_spans=None):
|
||||
assert text_threshold is not None or token_spans is not None, "text_threshould and token_spans should not be None at the same time!"
|
||||
caption = caption.lower()
|
||||
caption = caption.strip()
|
||||
if not caption.endswith("."):
|
||||
caption = caption + "."
|
||||
device = "cuda" if not cpu_only else "cpu"
|
||||
model = model.to(device)
|
||||
image = image.to(device)
|
||||
with torch.no_grad():
|
||||
outputs = model(image[None], captions=[caption])
|
||||
logits = outputs["pred_logits"].sigmoid()[0] # (nq, 256)
|
||||
boxes = outputs["pred_boxes"][0] # (nq, 4)
|
||||
|
||||
# filter output
|
||||
if token_spans is None:
|
||||
logits_filt = logits.cpu().clone()
|
||||
boxes_filt = boxes.cpu().clone()
|
||||
filt_mask = logits_filt.max(dim=1)[0] > box_threshold
|
||||
logits_filt = logits_filt[filt_mask] # num_filt, 256
|
||||
boxes_filt = boxes_filt[filt_mask] # num_filt, 4
|
||||
|
||||
# get phrase
|
||||
tokenlizer = model.tokenizer
|
||||
tokenized = tokenlizer(caption)
|
||||
# build pred
|
||||
pred_phrases = []
|
||||
for logit, box in zip(logits_filt, boxes_filt):
|
||||
pred_phrase = get_phrases_from_posmap(logit > text_threshold, tokenized, tokenlizer)
|
||||
if with_logits:
|
||||
pred_phrases.append(pred_phrase + f"({str(logit.max().item())[:4]})")
|
||||
else:
|
||||
pred_phrases.append(pred_phrase)
|
||||
else:
|
||||
# given-phrase mode
|
||||
positive_maps = create_positive_map_from_span(
|
||||
model.tokenizer(text_prompt),
|
||||
token_span=token_spans
|
||||
).to(image.device) # n_phrase, 256
|
||||
|
||||
logits_for_phrases = positive_maps @ logits.T # n_phrase, nq
|
||||
all_logits = []
|
||||
all_phrases = []
|
||||
all_boxes = []
|
||||
for (token_span, logit_phr) in zip(token_spans, logits_for_phrases):
|
||||
# get phrase
|
||||
phrase = ' '.join([caption[_s:_e] for (_s, _e) in token_span])
|
||||
# get mask
|
||||
filt_mask = logit_phr > box_threshold
|
||||
# filt box
|
||||
all_boxes.append(boxes[filt_mask])
|
||||
# filt logits
|
||||
all_logits.append(logit_phr[filt_mask])
|
||||
if with_logits:
|
||||
logit_phr_num = logit_phr[filt_mask]
|
||||
all_phrases.extend([phrase + f"({str(logit.item())[:4]})" for logit in logit_phr_num])
|
||||
else:
|
||||
all_phrases.extend([phrase for _ in range(len(filt_mask))])
|
||||
boxes_filt = torch.cat(all_boxes, dim=0).cpu()
|
||||
pred_phrases = all_phrases
|
||||
|
||||
|
||||
return boxes_filt, pred_phrases
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
parser = argparse.ArgumentParser("Grounding DINO example", add_help=True)
|
||||
parser.add_argument("--config_file", "-c", type=str, required=True, help="path to config file")
|
||||
parser.add_argument(
|
||||
"--checkpoint_path", "-p", type=str, required=True, help="path to checkpoint file"
|
||||
)
|
||||
parser.add_argument("--image_path", "-i", type=str, required=True, help="path to image file")
|
||||
parser.add_argument("--text_prompt", "-t", type=str, required=True, help="text prompt")
|
||||
parser.add_argument(
|
||||
"--output_dir", "-o", type=str, default="outputs", required=True, help="output directory"
|
||||
)
|
||||
|
||||
parser.add_argument("--box_threshold", type=float, default=0.3, help="box threshold")
|
||||
parser.add_argument("--text_threshold", type=float, default=0.25, help="text threshold")
|
||||
parser.add_argument("--token_spans", type=str, default=None, help=
|
||||
"The positions of start and end positions of phrases of interest. \
|
||||
For example, a caption is 'a cat and a dog', \
|
||||
if you would like to detect 'cat', the token_spans should be '[[[2, 5]], ]', since 'a cat and a dog'[2:5] is 'cat'. \
|
||||
if you would like to detect 'a cat', the token_spans should be '[[[0, 1], [2, 5]], ]', since 'a cat and a dog'[0:1] is 'a', and 'a cat and a dog'[2:5] is 'cat'. \
|
||||
")
|
||||
|
||||
parser.add_argument("--cpu-only", action="store_true", help="running on cpu only!, default=False")
|
||||
args = parser.parse_args()
|
||||
|
||||
# cfg
|
||||
config_file = args.config_file # change the path of the model config file
|
||||
checkpoint_path = args.checkpoint_path # change the path of the model
|
||||
image_path = args.image_path
|
||||
text_prompt = args.text_prompt
|
||||
output_dir = args.output_dir
|
||||
box_threshold = args.box_threshold
|
||||
text_threshold = args.text_threshold
|
||||
token_spans = args.token_spans
|
||||
|
||||
# make dir
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
# load image
|
||||
image_pil, image = load_image(image_path)
|
||||
# load model
|
||||
model = load_model(config_file, checkpoint_path, cpu_only=args.cpu_only)
|
||||
|
||||
# visualize raw image
|
||||
image_pil.save(os.path.join(output_dir, "raw_image.jpg"))
|
||||
|
||||
# set the text_threshold to None if token_spans is set.
|
||||
if token_spans is not None:
|
||||
text_threshold = None
|
||||
print("Using token_spans. Set the text_threshold to None.")
|
||||
|
||||
|
||||
# run model
|
||||
boxes_filt, pred_phrases = get_grounding_output(
|
||||
model, image, text_prompt, box_threshold, text_threshold, cpu_only=args.cpu_only, token_spans=eval(token_spans)
|
||||
)
|
||||
|
||||
# visualize pred
|
||||
size = image_pil.size
|
||||
pred_dict = {
|
||||
"boxes": boxes_filt,
|
||||
"size": [size[1], size[0]], # H,W
|
||||
"labels": pred_phrases,
|
||||
}
|
||||
# import ipdb; ipdb.set_trace()
|
||||
image_with_box = plot_boxes_to_image(image_pil, pred_dict)[0]
|
||||
image_with_box.save(os.path.join(output_dir, "pred.jpg"))
|
||||
@ -1,230 +0,0 @@
|
||||
import argparse
|
||||
import time
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from groundingdino.models import build_model
|
||||
import groundingdino.datasets.transforms as T
|
||||
from groundingdino.util import box_ops, get_tokenlizer
|
||||
from groundingdino.util.misc import clean_state_dict, collate_fn
|
||||
from groundingdino.util.slconfig import SLConfig
|
||||
|
||||
# from torchvision.datasets import CocoDetection
|
||||
import torchvision
|
||||
|
||||
from groundingdino.util.vl_utils import build_captions_and_token_span, create_positive_map_from_span
|
||||
from groundingdino.datasets.cocogrounding_eval import CocoGroundingEvaluator
|
||||
|
||||
|
||||
def load_model(model_config_path: str, model_checkpoint_path: str, device: str = "cuda"):
|
||||
args = SLConfig.fromfile(model_config_path)
|
||||
args.device = device
|
||||
model = build_model(args)
|
||||
checkpoint = torch.load(model_checkpoint_path, map_location="cpu")
|
||||
model.load_state_dict(clean_state_dict(checkpoint["ema_model"]), strict=False)
|
||||
model.eval()
|
||||
return model
|
||||
|
||||
|
||||
class CocoDetection(torchvision.datasets.CocoDetection):
|
||||
def __init__(self, img_folder, ann_file, transforms):
|
||||
super().__init__(img_folder, ann_file)
|
||||
self._transforms = transforms
|
||||
|
||||
def __getitem__(self, idx):
|
||||
img, target = super().__getitem__(idx) # target: list
|
||||
|
||||
# import ipdb; ipdb.set_trace()
|
||||
|
||||
w, h = img.size
|
||||
boxes = [obj["bbox"] for obj in target]
|
||||
boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4)
|
||||
boxes[:, 2:] += boxes[:, :2] # xywh -> xyxy
|
||||
boxes[:, 0::2].clamp_(min=0, max=w)
|
||||
boxes[:, 1::2].clamp_(min=0, max=h)
|
||||
# filt invalid boxes/masks/keypoints
|
||||
keep = (boxes[:, 3] > boxes[:, 1]) & (boxes[:, 2] > boxes[:, 0])
|
||||
boxes = boxes[keep]
|
||||
|
||||
target_new = {}
|
||||
image_id = self.ids[idx]
|
||||
target_new["image_id"] = image_id
|
||||
target_new["boxes"] = boxes
|
||||
target_new["orig_size"] = torch.as_tensor([int(h), int(w)])
|
||||
|
||||
if self._transforms is not None:
|
||||
img, target = self._transforms(img, target_new)
|
||||
|
||||
return img, target
|
||||
|
||||
|
||||
class PostProcessCocoGrounding(nn.Module):
|
||||
""" This module converts the model's output into the format expected by the coco api"""
|
||||
|
||||
def __init__(self, num_select=300, coco_api=None, tokenlizer=None) -> None:
|
||||
super().__init__()
|
||||
self.num_select = num_select
|
||||
|
||||
assert coco_api is not None
|
||||
category_dict = coco_api.dataset['categories']
|
||||
cat_list = [item['name'] for item in category_dict]
|
||||
captions, cat2tokenspan = build_captions_and_token_span(cat_list, True)
|
||||
tokenspanlist = [cat2tokenspan[cat] for cat in cat_list]
|
||||
positive_map = create_positive_map_from_span(
|
||||
tokenlizer(captions), tokenspanlist) # 80, 256. normed
|
||||
|
||||
id_map = {0: 1, 1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7, 7: 8, 8: 9, 9: 10, 10: 11, 11: 13, 12: 14, 13: 15, 14: 16, 15: 17, 16: 18, 17: 19, 18: 20, 19: 21, 20: 22, 21: 23, 22: 24, 23: 25, 24: 27, 25: 28, 26: 31, 27: 32, 28: 33, 29: 34, 30: 35, 31: 36, 32: 37, 33: 38, 34: 39, 35: 40, 36: 41, 37: 42, 38: 43, 39: 44, 40: 46,
|
||||
41: 47, 42: 48, 43: 49, 44: 50, 45: 51, 46: 52, 47: 53, 48: 54, 49: 55, 50: 56, 51: 57, 52: 58, 53: 59, 54: 60, 55: 61, 56: 62, 57: 63, 58: 64, 59: 65, 60: 67, 61: 70, 62: 72, 63: 73, 64: 74, 65: 75, 66: 76, 67: 77, 68: 78, 69: 79, 70: 80, 71: 81, 72: 82, 73: 84, 74: 85, 75: 86, 76: 87, 77: 88, 78: 89, 79: 90}
|
||||
|
||||
# build a mapping from label_id to pos_map
|
||||
new_pos_map = torch.zeros((91, 256))
|
||||
for k, v in id_map.items():
|
||||
new_pos_map[v] = positive_map[k]
|
||||
self.positive_map = new_pos_map
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, outputs, target_sizes, not_to_xyxy=False):
|
||||
""" Perform the computation
|
||||
Parameters:
|
||||
outputs: raw outputs of the model
|
||||
target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch
|
||||
For evaluation, this must be the original image size (before any data augmentation)
|
||||
For visualization, this should be the image size after data augment, but before padding
|
||||
"""
|
||||
num_select = self.num_select
|
||||
out_logits, out_bbox = outputs['pred_logits'], outputs['pred_boxes']
|
||||
|
||||
# pos map to logit
|
||||
prob_to_token = out_logits.sigmoid() # bs, 100, 256
|
||||
pos_maps = self.positive_map.to(prob_to_token.device)
|
||||
# (bs, 100, 256) @ (91, 256).T -> (bs, 100, 91)
|
||||
prob_to_label = prob_to_token @ pos_maps.T
|
||||
|
||||
# if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO':
|
||||
# import ipdb; ipdb.set_trace()
|
||||
|
||||
assert len(out_logits) == len(target_sizes)
|
||||
assert target_sizes.shape[1] == 2
|
||||
|
||||
prob = prob_to_label
|
||||
topk_values, topk_indexes = torch.topk(
|
||||
prob.view(out_logits.shape[0], -1), num_select, dim=1)
|
||||
scores = topk_values
|
||||
topk_boxes = topk_indexes // prob.shape[2]
|
||||
labels = topk_indexes % prob.shape[2]
|
||||
|
||||
if not_to_xyxy:
|
||||
boxes = out_bbox
|
||||
else:
|
||||
boxes = box_ops.box_cxcywh_to_xyxy(out_bbox)
|
||||
|
||||
boxes = torch.gather(
|
||||
boxes, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4))
|
||||
|
||||
# and from relative [0, 1] to absolute [0, height] coordinates
|
||||
img_h, img_w = target_sizes.unbind(1)
|
||||
scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1)
|
||||
boxes = boxes * scale_fct[:, None, :]
|
||||
|
||||
results = [{'scores': s, 'labels': l, 'boxes': b}
|
||||
for s, l, b in zip(scores, labels, boxes)]
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main(args):
|
||||
# config
|
||||
cfg = SLConfig.fromfile(args.config_file)
|
||||
|
||||
# build model
|
||||
model = load_model(args.config_file, args.checkpoint_path)
|
||||
model = model.to(args.device)
|
||||
model = model.eval()
|
||||
|
||||
# build dataloader
|
||||
transform = T.Compose(
|
||||
[
|
||||
T.RandomResize([800], max_size=1333),
|
||||
T.ToTensor(),
|
||||
T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
|
||||
]
|
||||
)
|
||||
dataset = CocoDetection(
|
||||
args.image_dir, args.anno_path, transforms=transform)
|
||||
data_loader = DataLoader(
|
||||
dataset, batch_size=1, shuffle=False, num_workers=args.num_workers, collate_fn=collate_fn)
|
||||
|
||||
# build post processor
|
||||
tokenlizer = get_tokenlizer.get_tokenlizer(cfg.text_encoder_type)
|
||||
postprocessor = PostProcessCocoGrounding(
|
||||
coco_api=dataset.coco, tokenlizer=tokenlizer)
|
||||
|
||||
# build evaluator
|
||||
evaluator = CocoGroundingEvaluator(
|
||||
dataset.coco, iou_types=("bbox",), useCats=True)
|
||||
|
||||
# build captions
|
||||
category_dict = dataset.coco.dataset['categories']
|
||||
cat_list = [item['name'] for item in category_dict]
|
||||
caption = " . ".join(cat_list) + ' .'
|
||||
print("Input text prompt:", caption)
|
||||
|
||||
# run inference
|
||||
start = time.time()
|
||||
for i, (images, targets) in enumerate(data_loader):
|
||||
# get images and captions
|
||||
images = images.tensors.to(args.device)
|
||||
bs = images.shape[0]
|
||||
input_captions = [caption] * bs
|
||||
|
||||
# feed to the model
|
||||
outputs = model(images, captions=input_captions)
|
||||
|
||||
orig_target_sizes = torch.stack(
|
||||
[t["orig_size"] for t in targets], dim=0).to(images.device)
|
||||
results = postprocessor(outputs, orig_target_sizes)
|
||||
cocogrounding_res = {
|
||||
target["image_id"]: output for target, output in zip(targets, results)}
|
||||
evaluator.update(cocogrounding_res)
|
||||
|
||||
if (i+1) % 30 == 0:
|
||||
used_time = time.time() - start
|
||||
eta = len(data_loader) / (i+1e-5) * used_time - used_time
|
||||
print(
|
||||
f"processed {i}/{len(data_loader)} images. time: {used_time:.2f}s, ETA: {eta:.2f}s")
|
||||
|
||||
evaluator.synchronize_between_processes()
|
||||
evaluator.accumulate()
|
||||
evaluator.summarize()
|
||||
|
||||
print("Final results:", evaluator.coco_eval["bbox"].stats.tolist())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
"Grounding DINO eval on COCO", add_help=True)
|
||||
# load model
|
||||
parser.add_argument("--config_file", "-c", type=str,
|
||||
required=True, help="path to config file")
|
||||
parser.add_argument(
|
||||
"--checkpoint_path", "-p", type=str, required=True, help="path to checkpoint file"
|
||||
)
|
||||
parser.add_argument("--device", type=str, default="cuda",
|
||||
help="running device (default: cuda)")
|
||||
|
||||
# post processing
|
||||
parser.add_argument("--num_select", type=int, default=300,
|
||||
help="number of topk to select")
|
||||
|
||||
# coco info
|
||||
parser.add_argument("--anno_path", type=str,
|
||||
required=True, help="coco root")
|
||||
parser.add_argument("--image_dir", type=str,
|
||||
required=True, help="coco image dir")
|
||||
parser.add_argument("--num_workers", type=int, default=4,
|
||||
help="number of workers for dataloader")
|
||||
args = parser.parse_args()
|
||||
|
||||
main(args)
|
||||
@ -1,10 +0,0 @@
|
||||
torch
|
||||
torchvision
|
||||
transformers
|
||||
addict
|
||||
yapf
|
||||
timm
|
||||
numpy
|
||||
opencv-python
|
||||
supervision==0.6.0
|
||||
pycocotools
|
||||
@ -1,208 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2022 The IDEA Authors. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# Modified from
|
||||
# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/setup.py
|
||||
# https://github.com/facebookresearch/detectron2/blob/main/setup.py
|
||||
# https://github.com/open-mmlab/mmdetection/blob/master/setup.py
|
||||
# https://github.com/Oneflow-Inc/libai/blob/main/setup.py
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
|
||||
import glob
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import torch
|
||||
from setuptools import find_packages, setup
|
||||
from torch.utils.cpp_extension import CUDA_HOME, CppExtension, CUDAExtension
|
||||
|
||||
# groundingdino version info
|
||||
version = "0.1.0"
|
||||
package_name = "groundingdino"
|
||||
cwd = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
sha = "Unknown"
|
||||
try:
|
||||
sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd).decode("ascii").strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def write_version_file():
|
||||
version_path = os.path.join(cwd, "groundingdino", "version.py")
|
||||
with open(version_path, "w") as f:
|
||||
f.write(f"__version__ = '{version}'\n")
|
||||
# f.write(f"git_version = {repr(sha)}\n")
|
||||
|
||||
|
||||
requirements = ["torch", "torchvision"]
|
||||
|
||||
torch_ver = [int(x) for x in torch.__version__.split(".")[:2]]
|
||||
|
||||
|
||||
def get_extensions():
|
||||
this_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
extensions_dir = os.path.join(this_dir, "groundingdino", "models", "GroundingDINO", "csrc")
|
||||
|
||||
main_source = os.path.join(extensions_dir, "vision.cpp")
|
||||
sources = glob.glob(os.path.join(extensions_dir, "**", "*.cpp"))
|
||||
source_cuda = glob.glob(os.path.join(extensions_dir, "**", "*.cu")) + glob.glob(
|
||||
os.path.join(extensions_dir, "*.cu")
|
||||
)
|
||||
|
||||
sources = [main_source] + sources
|
||||
|
||||
extension = CppExtension
|
||||
|
||||
extra_compile_args = {"cxx": []}
|
||||
define_macros = []
|
||||
|
||||
if CUDA_HOME is not None and (torch.cuda.is_available() or "TORCH_CUDA_ARCH_LIST" in os.environ):
|
||||
print("Compiling with CUDA")
|
||||
extension = CUDAExtension
|
||||
sources += source_cuda
|
||||
define_macros += [("WITH_CUDA", None)]
|
||||
extra_compile_args["nvcc"] = [
|
||||
"-DCUDA_HAS_FP16=1",
|
||||
"-D__CUDA_NO_HALF_OPERATORS__",
|
||||
"-D__CUDA_NO_HALF_CONVERSIONS__",
|
||||
"-D__CUDA_NO_HALF2_OPERATORS__",
|
||||
]
|
||||
else:
|
||||
print("Compiling without CUDA")
|
||||
define_macros += [("WITH_HIP", None)]
|
||||
extra_compile_args["nvcc"] = []
|
||||
return None
|
||||
|
||||
sources = [os.path.join(extensions_dir, s) for s in sources]
|
||||
include_dirs = [extensions_dir]
|
||||
|
||||
ext_modules = [
|
||||
extension(
|
||||
"groundingdino._C",
|
||||
sources,
|
||||
include_dirs=include_dirs,
|
||||
define_macros=define_macros,
|
||||
extra_compile_args=extra_compile_args,
|
||||
)
|
||||
]
|
||||
|
||||
return ext_modules
|
||||
|
||||
|
||||
def parse_requirements(fname="requirements.txt", with_version=True):
|
||||
"""Parse the package dependencies listed in a requirements file but strips
|
||||
specific versioning information.
|
||||
|
||||
Args:
|
||||
fname (str): path to requirements file
|
||||
with_version (bool, default=False): if True include version specs
|
||||
|
||||
Returns:
|
||||
List[str]: list of requirements items
|
||||
|
||||
CommandLine:
|
||||
python -c "import setup; print(setup.parse_requirements())"
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
from os.path import exists
|
||||
|
||||
require_fpath = fname
|
||||
|
||||
def parse_line(line):
|
||||
"""Parse information from a line in a requirements text file."""
|
||||
if line.startswith("-r "):
|
||||
# Allow specifying requirements in other files
|
||||
target = line.split(" ")[1]
|
||||
for info in parse_require_file(target):
|
||||
yield info
|
||||
else:
|
||||
info = {"line": line}
|
||||
if line.startswith("-e "):
|
||||
info["package"] = line.split("#egg=")[1]
|
||||
elif "@git+" in line:
|
||||
info["package"] = line
|
||||
else:
|
||||
# Remove versioning from the package
|
||||
pat = "(" + "|".join([">=", "==", ">"]) + ")"
|
||||
parts = re.split(pat, line, maxsplit=1)
|
||||
parts = [p.strip() for p in parts]
|
||||
|
||||
info["package"] = parts[0]
|
||||
if len(parts) > 1:
|
||||
op, rest = parts[1:]
|
||||
if ";" in rest:
|
||||
# Handle platform specific dependencies
|
||||
# http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies
|
||||
version, platform_deps = map(str.strip, rest.split(";"))
|
||||
info["platform_deps"] = platform_deps
|
||||
else:
|
||||
version = rest # NOQA
|
||||
info["version"] = (op, version)
|
||||
yield info
|
||||
|
||||
def parse_require_file(fpath):
|
||||
with open(fpath, "r") as f:
|
||||
for line in f.readlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
for info in parse_line(line):
|
||||
yield info
|
||||
|
||||
def gen_packages_items():
|
||||
if exists(require_fpath):
|
||||
for info in parse_require_file(require_fpath):
|
||||
parts = [info["package"]]
|
||||
if with_version and "version" in info:
|
||||
parts.extend(info["version"])
|
||||
if not sys.version.startswith("3.4"):
|
||||
# apparently package_deps are broken in 3.4
|
||||
platform_deps = info.get("platform_deps")
|
||||
if platform_deps is not None:
|
||||
parts.append(";" + platform_deps)
|
||||
item = "".join(parts)
|
||||
yield item
|
||||
|
||||
packages = list(gen_packages_items())
|
||||
return packages
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f"Building wheel {package_name}-{version}")
|
||||
|
||||
with open("LICENSE", "r", encoding="utf-8") as f:
|
||||
license = f.read()
|
||||
|
||||
write_version_file()
|
||||
|
||||
setup(
|
||||
name="groundingdino",
|
||||
version="0.1.0",
|
||||
author="International Digital Economy Academy, Shilong Liu",
|
||||
url="https://github.com/IDEA-Research/GroundingDINO",
|
||||
description="open-set object detector",
|
||||
license=license,
|
||||
install_requires=parse_requirements("requirements.txt"),
|
||||
packages=find_packages(
|
||||
exclude=(
|
||||
"configs",
|
||||
"tests",
|
||||
)
|
||||
),
|
||||
ext_modules=get_extensions(),
|
||||
cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension},
|
||||
)
|
||||
Loading…
Reference in new issue