Add support for SAM 2

[giswqs]

This PR adds support for Meta's Segment Anything Model 2 (SAM 2).
https://github.com/facebookresearch/segment-anything-2
https://github.com/IDEA-Research/Grounded-SAM-2?tab=readme-ov-file#grounded-sam-2-video-object-tracking-demo

Work in progress

[github-actions]

🚀 Deployed on https://66e87055e71a803c2aff3721--opengeos.netlify.app

[giswqs]

from samgeo import SamGeo2

sam2 = SamGeo2(model_id="sam2-hiera-large", automatic=True)
image = "images/cars.jpg"
masks = sam2.generate(image)

[giswqs]

from samgeo import SamGeo2

sam2 = SamGeo2(
    model_id="sam2-hiera-large",
    apply_postprocessing=False,
    points_per_side=32,
    points_per_batch=64,
    pred_iou_thresh=0.7,
    stability_score_thresh=0.92,
    stability_score_offset=0.7,
    crop_n_layers=1,
    box_nms_thresh=0.7,
    crop_n_points_downscale_factor=2,
    min_mask_region_area=25.0,
    use_m2m=True,
)
image = "images/satellite.tif"
masks = sam2.generate(image)

[giswqs]

Peek.2024-09-13.15-56.mp4

[twoldemariam]

What an exciting time we are in!

[brendancol]

• Support SAM2 in LangSAM (https://github.com/opengeos/segment-geospatial/blob/main/samgeo/text_sam.py#L134-L150)

[giswqs]

@brendancol Yes, we should be able to add support for LangSAM.
The Grounded SAM2 is also very interesting.

[giswqs]

from samgeo import SamGeo2

predictor = SamGeo2(video=True)
video_path = "input_video.mp4"
predictor.set_video(video_path)
prompts = {
    1: {
        "points": [[200, 300], [275, 175]],
        "labels": [1, 0],
        "frame_idx": 0,
    },
    2: {
        "points": [[400, 150]],
        "labels": [1],
        "frame_idx": 0,
    },
}
predictor.predict_video(prompts)
predictor.save_video_segments_blended(
    "blended", output_video="output_video.mp4", fps=30
)

input video

input_video.mp4

output video

output_video.mp4

[giswqs]

from samgeo import SamGeo2

predictor = SamGeo2(video=True)
video_path = "input_video.mp4"
predictor.set_video(video_path, frame_rate=3)
prompts = {
    1: {
        "points": [[200, 300], [275, 175]],
        "labels": [1, 0],
        "frame_idx": 0,
    },
    2: {
        "points": [[400, 150]],
        "labels": [1],
        "frame_idx": 0,
    },
}
predictor.show_prompts(prompts, frame_idx=0)

[giswqs]

from samgeo import SamGeo2

predictor = SamGeo2(video=True)
video_path = "input_video.mp4"
predictor.set_video(video_path, frame_rate=3)
predictor.show_images()

[giswqs]

Input video

landsat.mp4

Point prompts

Output video

output_video2.mp4

[giswqs]

from samgeo import SamGeo2

predictor = SamGeo2(video=True)
video_path = "landsat_ts"
predictor.set_video(video_path)
prompts = {
    1: {
        "points": [[1582, 933], [1287, 905], [1473, 998]],
        "labels": [1, 1, 1],
        "frame_idx": 0,
    },
}
predictor.show_prompts(prompts, frame_idx=0)
predictor.predict_video(prompts)
predictor.save_video_segments("segments")

Input GeoTIFFs

landsat_images.mp4

Point prompts

Output GeoTIFFs

landsat_water.mp4

Blended mode

landsat_water_blended.mp4

[giswqs]

Input points can be either XY or lon/lat pairs.

from samgeo import SamGeo2

predictor = SamGeo2(video=True)
video_path = "landsat_ts"
predictor.set_video(video_path)
prompts = {
    1: {
        "points": [[-74.3713, -8.5218], [-74.2973, -8.5306], [-74.3230, -8.5495]],
        "labels": [1, 1, 1],
        "frame_idx": 0,
    },
}
predictor.show_prompts(prompts, frame_idx=0, point_crs="EPSG:4326")
predictor.predict_video()
predictor.save_video_segments_blended(
    "blended", fps=5, output_video="landsat_water_blended.mp4"
)