# vlm-binding

#### Counting task
```
# Generate counting trials with randomly colored circles. Saves results and metadata to output_dir.
python task_scripts/gen_vlm_counting.py --object_inds=37 \
                                        --n_trials=100 \
                                        --sigma=1 \
                                        --n_shapes 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 \
                                        --size=40 \
                                        --output_dir=data/vlm/counting_colored

# Generate counting trials with colored circles with low variance. Saves results and metadata to output_dir.
python task_scripts/gen_vlm_counting.py --object_inds=37 \
                                        --n_trials=100 \
                                        --sigma=0.01 \
                                        --n_shapes 1 2 3 4 5 6 7 8 9 10 \
                                        --size=45 \
                                        --output_dir=data/vlm/counting_low_variance

# Generate counting trials with only black circles. Saves results and metadata to output_dir.
python task_scripts/gen_vlm_counting.py --object_inds=37 \
                                        --n_trials=100 \
                                        --n_shapes 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 \
                                        --size=40 \
                                        --all_black=True \
                                        --output_dir=data/vlm/counting_black

# Run model on colored counting trials.
python run_vlm_requests.py --task='counting' \
                           --task_dir='data/vlm/counting_colored' \
                           --task_file='data/vlm/counting_colored/metadata.csv' \
                           --task_prompt_path='prompts/run/counting.txt' \
                           --parse_prompt_path='prompts/parse/counting.txt' \
                           --results_file='output/vlm_counting_colored.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=500 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10 \

# Run model on black counting trials.
python run_vlm_requests.py --task='counting' \
                           --task_dir='data/vlm/counting_black' \
                           --task_file='data/vlm/counting_black/metadata.csv' \
                           --task_prompt_path='prompts/run/counting.txt' \
                           --parse_prompt_path='prompts/parse/counting.txt' \
                           --results_file='output/vlm_counting_black.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=500 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10 
```

#### Popout task.
```
# Generate popout trials with only green and (one) red circles.
python task_scripts/gen_vlm_popout.py --n_objects 5 10 15 20 25 30 35 40 45 50 \
                                      --n_trials=100 \
                                      --size=24 \
                                      --colors green red \
                                      --shape_inds 37 \
                                      --output_dir=data/vlm/popout

# Generate popout trials with circles of any colors.
python task_scripts/gen_vlm_popout.py --n_objects 5 10 15 20 25 30 35 40 45 50 \
                                      --n_trials=100 \
                                      --size=22 \
                                      --shape_inds 37 \
                                      --output_dir=data/vlm/popout_complex

# Run models on popout trials.
python run_vlm_requests.py --task='popout' \
                           --task_dir='data/vlm/popout' \
                           --task_file='data/vlm/popout/metadata.csv' \
                           --task_prompt_path='prompts/run/popout.txt' \
                           --parse_prompt_path='prompts/parse/popout.txt' \
                           --results_file='output/vlm_popout.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=500 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10
```

#### Serial search task.
```
# Generate serial search trials with only green and red letters (T's and L's).
python task_scripts/gen_vlm_search.py --n_objects 5 10 15 20 25 30 35 40 45 50 \
                                      --n_trials=100 \
                                      --size=22 \
                                      --use_letters=True \
                                      --colors green red \
                                      --output_dir=data/vlm/search

# Run models on serial search trials.
python run_vlm_requests.py --task='search' \
                           --task_dir='data/vlm/search' \
                           --task_file='data/vlm/search/metadata.csv' \
                           --task_prompt_path='prompts/run/search.txt' \
                           --parse_prompt_path='prompts/parse/search.txt' \
                           --results_file='output/vlm_search.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=500 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10 \
```

#### Binding task.
```
# Generate binding trials with 10 objects.
python task_scripts/gen_vlm_binding.py --n_objects 2 3 4 5 6 7 8 9 10 \
                                       --n_trials=100 \
                                       --size=42 \
                                       --color_names red green blue gold purple saddlebrown gray black cyan darkorange \
                                       --shape_names airplane triangle cloud cross umbrella scissors heart star circle square \
                                       --shape_inds 6 9 21 24 34 60 96 98 100 101 \
                                       --output_dir=data/vlm/binding

# Old version of the binding task.
python task_scripts/gen_vlm_binding.py --n_objects 2 3 4 5 6 7 8 9 10 \
                                       --size=42 \
                                       --color_names red green blue gold purple saddlebrown gray black cyan darkorange \                             
                                       --shape_names airplane triangle cloud cross down-arrow umbrella pentagon scissors heart star \                                         
                                       --shape_inds 6 9 21 24 28 34 59 60 96 98 \
                                       --output_dir=data/vlm/binding

# Run model on binding trials.
python run_vlm_requests.py --task='binding' \
                           --task_dir='data/vlm/binding' \
                           --task_file='data/vlm/binding/metadata.csv' \
                           --task_prompt_path='prompts/run/binding.txt' \
                           --parse_prompt_path='prompts/parse/binding.txt' \
                           --results_file='output/vlm_binding.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=500 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10 \
```


#### RMTS task.
```
python task_scripts/gen_vlm_rmts.py --n_trials=100 \
                                    --color_names red green blue purple saddlebrown black \
                                    --shape_names triangle star heart cross circle square \
                                    --shape_inds 9 98 96 24 100 101 \
                                    --output_dir='data/vlm/rmts'

# Decomposed condition feature decoding
python run_vlm_requests.py --task='rmts_decomposed' \
                           --task_dir='data/vlm/rmts' \
                           --task_file='data/vlm/rmts/feature_task_metadata.csv' \
                           --task_prompt_path='prompts/run/rmts_features_decomposed.txt' \
                           --parse_prompt_path='prompts/parse/rmts_feature_task.txt' \
                           --results_file='output/rmts_decomposed_feature_decoding.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=10 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10 \
                           --sleep=10

# Unified condition feature decoding
python run_vlm_requests.py --task='rmts_unified' \
                           --task_dir='data/vlm/rmts' \
                           --task_file='data/vlm/rmts/feature_task_metadata.csv' \
                           --task_prompt_path='prompts/run/rmts_features_unified.txt' \
                           --parse_prompt_path='prompts/parse/rmts_feature_task.txt' \
                           --results_file='output/rmts_unified_feature_decoding.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=10 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10 \
                           --sleep=10

# Decomposed condition relation decoding
python run_vlm_requests.py --task='rmts_decomposed' \
                           --task_dir='data/vlm/rmts' \
                           --task_file='data/vlm/rmts/relation_task_metadata.csv' \
                           --task_prompt_path='prompts/run/rmts_relation_decomposed.txt' \
                           --parse_prompt_path='prompts/parse/rmts_relation_task.txt' \
                           --results_file='output/rmts_decomposed_relation_decoding.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=10 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10 \
                           --sleep=10

# Unified condition feature decoding
python run_vlm_requests.py --task='rmts_unified' \
                           --task_dir='data/vlm/rmts' \
                           --task_file='data/vlm/rmts/feature_task_metadata.csv' \
                           --task_prompt_path='prompts/run/rmts_features_unified.txt' \
                           --parse_prompt_path='prompts/parse/rmts_feature_task.txt' \
                           --results_file='output/rmts_unified_feature_decoding.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=10 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10 \
                           --sleep=10

# Decomposed condition full task
python run_vlm_requests.py --task='rmts_decomposed' \
                           --task_dir='data/vlm/rmts' \
                           --task_file='data/vlm/rmts/full_task_metadata.csv' \
                           --task_prompt_path='prompts/run/rmts_full_decomposed.txt' \
                           --parse_prompt_path='prompts/parse/rmts_full_task.txt' \
                           --results_file='output/rmts_decomposed_full_task.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=500 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10 \
                           --sleep=10

# Unified condition full task
python run_vlm_requests.py --task='rmts_unified' \
                           --task_dir='data/vlm/rmts' \
                           --task_file='data/vlm/rmts/full_task_metadata.csv' \
                           --task_prompt_path='prompts/run/rmts_full_unified.txt' \
                           --parse_prompt_path='prompts/parse/rmts_full_task.txt' \
                           --results_file='output/rmts_unified_full_task.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/gpt4v_image.json' \
                           --parse_payload='payloads/gpt4_parse.json' \
                           --max_vision_tokens=500 \
                           --max_parse_tokens=10 \
                           --api='azure' \
                           --log_interval=10 \
                           --sleep=10
```

#### TTI counting.
```
python task_scripts/gen_tti_counting.py --n_shapes 2 3 4 5 6 7 8 9 10 \
                                        --objects sphere cone cube cylinder \
                                        --n_trials=100 \
                                        --output_dir='data/tti/counting'

python run_tti_requests.py --task=counting \
                           --task_file='data/tti/counting/metadata.csv' \
                           --results_file='output/tti_counting.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/dalle3.json' \
                           --api='dalle' \
                           --n_trials=10 \
                           --log_interval=10 \
                           --sleep=0
```

#### TTI binding.
```
python task_scripts/gen_tti_binding.py --n_objects 2 3 4 \
                                       --object_names teacup balloon candle hat pen \
                                       --color_names red green blue yellow black \
                                       --n_trials=100 \
                                       --output_dir='data/tti/binding'

python run_tti_requests.py --task=binding \
                           --task_file='data/tti/binding/metadata.csv' \
                           --results_file='output/tti_binding.csv' \
                           --api_file='api_metadata.json' \
                           --task_payload='payloads/dalle3.json' \
                           --api='dalle' \
                           --log_interval=10 \
                           --sleep=0
```