Gemini 模型从一开始就具有多模态特性,可解锁各种图片处理和计算机视觉任务,包括但不限于图片说明、分类和视觉问答,而无需训练专门的机器学习模型。
将图片传递给 Gemini
您可以通过以下两种方法将图片作为输入提供给 Gemini:
- 传递内嵌图片数据:非常适合较小的文件(请求总大小小于 20MB,包括提示)。
- 使用 File API 上传图片:适用于较大的文件,或在多个请求中重复使用图片。
传递内嵌图片数据
您可以将请求中的内嵌图片数据传递给 generateContent
。您可以以 Base64 编码字符串的形式提供图片数据,也可以直接读取本地文件(具体取决于语言)。
以下示例展示了如何从本地文件读取图片,并将其传递给 generateContent
API 进行处理。
Python
from google.genai import types
with open('path/to/small-sample.jpg', 'rb') as f:
image_bytes = f.read()
response = client.models.generate_content(
model='gemini-2.0-flash',
contents=[
types.Part.from_bytes(
data=image_bytes,
mime_type='image/jpeg',
),
'Caption this image.'
]
)
print(response.text)
JavaScript
import { GoogleGenAI } from "@google/genai";
import * as fs from "node:fs";
const ai = new GoogleGenAI({ apiKey: "GOOGLE_API_KEY" });
const base64ImageFile = fs.readFileSync("path/to/small-sample.jpg", {
encoding: "base64",
});
const contents = [
{
inlineData: {
mimeType: "image/jpeg",
data: base64ImageFile,
},
},
{ text: "Caption this image." },
];
const response = await ai.models.generateContent({
model: "gemini-2.0-flash",
contents: contents,
});
console.log(response.text);
Go
bytes, _ := os.ReadFile("path/to/small-sample.jpg")
parts := []*genai.Part{
genai.NewPartFromBytes(bytes, "image/jpeg"),
genai.NewPartFromText("Caption this image."),
}
contents := []*genai.Content{
genai.NewContentFromParts(parts, genai.RoleUser),
}
result, _ := client.Models.GenerateContent(
ctx,
"gemini-2.0-flash",
contents,
nil,
)
fmt.Println(result.Text())
REST
IMG_PATH="/path/to/your/image1.jpg"
if [[ "$(base64 --version 2>&1)" = *"FreeBSD"* ]]; then
B64FLAGS="--input"
else
B64FLAGS="-w0"
fi
curl "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=$GOOGLE_API_KEY" \
-H 'Content-Type: application/json' \
-X POST \
-d '{
"contents": [{
"parts":[
{
"inline_data": {
"mime_type":"image/jpeg",
"data": "'"$(base64 $B64FLAGS $IMG_PATH)"'"
}
},
{"text": "Caption this image."},
]
}]
}' 2> /dev/null
您还可以从网址提取图片,将其转换为字节,然后将其传递给 generateContent
,如以下示例所示。
Python
from google import genai
from google.genai import types
import requests
image_path = "https://goo.gle/instrument-img"
image_bytes = requests.get(image_path).content
image = types.Part.from_bytes(
data=image_bytes, mime_type="image/jpeg"
)
client = genai.Client(api_key="GOOGLE_API_KEY")
response = client.models.generate_content(
model="gemini-2.0-flash-exp",
contents=["What is this image?", image],
)
print(response.text)
JavaScript
import { GoogleGenAI } from "@google/genai";
async function main() {
const ai = new GoogleGenAI({ apiKey: process.env.GOOGLE_API_KEY });
const imageUrl = "https://goo.gle/instrument-img";
const response = await fetch(imageUrl);
const imageArrayBuffer = await response.arrayBuffer();
const base64ImageData = Buffer.from(imageArrayBuffer).toString('base64');
const result = await ai.models.generateContent({
model: "gemini-2.0-flash",
contents: [
{
inlineData: {
mimeType: 'image/jpeg',
data: base64ImageData,
},
},
{ text: "Caption this image." }
],
});
console.log(result.text);
}
main();
Go
package main
import (
"context"
"fmt"
"os"
"io"
"net/http"
"google.golang.org/genai"
)
func main() {
ctx := context.Background()
client, _ := genai.NewClient(ctx, &genai.ClientConfig{
APIKey: os.Getenv("GOOGLE_API_KEY"),
Backend: genai.BackendGeminiAPI,
})
// Download the image.
imageResp, _ := http.Get("https://goo.gle/instrument-img")
imageBytes, _ := io.ReadAll(imageResp.Body)
parts := []*genai.Part{
genai.NewPartFromBytes(imageBytes, "image/jpeg"),
genai.NewPartFromText("Caption this image."),
}
contents := []*genai.Content{
genai.NewContentFromParts(parts, genai.RoleUser),
}
result, _ := client.Models.GenerateContent(
ctx,
"gemini-2.0-flash",
contents,
nil,
)
fmt.Println(result.Text())
}
REST
IMG_URL="https://goo.gle/instrument-img"
MIME_TYPE=$(curl -sIL "$IMG_URL" | grep -i '^content-type:' | awk -F ': ' '{print $2}' | sed 's/\r$//' | head -n 1)
if [[ -z "$MIME_TYPE" || ! "$MIME_TYPE" == image/* ]]; then
MIME_TYPE="image/jpeg"
fi
# Check for macOS
if [[ "$(uname)" == "Darwin" ]]; then
IMAGE_B64=$(curl -sL "$IMG_URL" | base64 -b 0)
elif [[ "$(base64 --version 2>&1)" = *"FreeBSD"* ]]; then
IMAGE_B64=$(curl -sL "$IMG_URL" | base64)
else
IMAGE_B64=$(curl -sL "$IMG_URL" | base64 -w0)
fi
curl "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=$GEMINI_API_KEY" \
-H 'Content-Type: application/json' \
-X POST \
-d '{
"contents": [{
"parts":[
{
"inline_data": {
"mime_type":"'"$MIME_TYPE"'",
"data": "'"$IMAGE_B64"'"
}
},
{"text": "Caption this image."}
]
}]
}' 2> /dev/null
使用 File API 上传图片
对于大型文件或需要反复使用同一图片文件,请使用 Files API。以下代码会上传图片文件,然后在对 generateContent
的调用中使用该文件。如需了解详情和示例,请参阅 Files API 指南。
Python
from google import genai
client = genai.Client(api_key="GOOGLE_API_KEY")
my_file = client.files.upload(file="path/to/sample.jpg")
response = client.models.generate_content(
model="gemini-2.0-flash",
contents=[my_file, "Caption this image."],
)
print(response.text)
JavaScript
import {
GoogleGenAI,
createUserContent,
createPartFromUri,
} from "@google/genai";
const ai = new GoogleGenAI({ apiKey: "GOOGLE_API_KEY" });
async function main() {
const myfile = await ai.files.upload({
file: "path/to/sample.jpg",
config: { mimeType: "image/jpeg" },
});
const response = await ai.models.generateContent({
model: "gemini-2.0-flash",
contents: createUserContent([
createPartFromUri(myfile.uri, myfile.mimeType),
"Caption this image.",
]),
});
console.log(response.text);
}
await main();
Go
package main
import (
"context"
"fmt"
"os"
"google.golang.org/genai"
)
func main() {
ctx := context.Background()
client, _ := genai.NewClient(ctx, &genai.ClientConfig{
APIKey: os.Getenv("GOOGLE_API_KEY"),
Backend: genai.BackendGeminiAPI,
})
uploadedFile, _ := client.Files.UploadFromPath(ctx, "path/to/sample.jpg", nil)
parts := []*genai.Part{
genai.NewPartFromText("Caption this image."),
genai.NewPartFromURI(uploadedFile.URI, uploadedFile.MIMEType),
}
contents := []*genai.Content{
genai.NewContentFromParts(parts, genai.RoleUser),
}
result, _ := client.Models.GenerateContent(
ctx,
"gemini-2.0-flash",
contents,
nil,
)
fmt.Println(result.Text())
}
REST
IMAGE_PATH="path/to/sample.jpg"
MIME_TYPE=$(file -b --mime-type "${IMAGE_PATH}")
NUM_BYTES=$(wc -c < "${IMAGE_PATH}")
DISPLAY_NAME=IMAGE
tmp_header_file=upload-header.tmp
# Initial resumable request defining metadata.
# The upload url is in the response headers dump them to a file.
curl "https://generativelanguage.googleapis.com/upload/v1beta/files?key=${GOOGLE_API_KEY}" \
-D upload-header.tmp \
-H "X-Goog-Upload-Protocol: resumable" \
-H "X-Goog-Upload-Command: start" \
-H "X-Goog-Upload-Header-Content-Length: ${NUM_BYTES}" \
-H "X-Goog-Upload-Header-Content-Type: ${MIME_TYPE}" \
-H "Content-Type: application/json" \
-d "{'file': {'display_name': '${DISPLAY_NAME}'}}" 2> /dev/null
upload_url=$(grep -i "x-goog-upload-url: " "${tmp_header_file}" | cut -d" " -f2 | tr -d "\r")
rm "${tmp_header_file}"
# Upload the actual bytes.
curl "${upload_url}" \
-H "Content-Length: ${NUM_BYTES}" \
-H "X-Goog-Upload-Offset: 0" \
-H "X-Goog-Upload-Command: upload, finalize" \
--data-binary "@${IMAGE_PATH}" 2> /dev/null > file_info.json
file_uri=$(jq -r ".file.uri" file_info.json)
echo file_uri=$file_uri
# Now generate content using that file
curl "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=$GOOGLE_API_KEY" \
-H 'Content-Type: application/json' \
-X POST \
-d '{
"contents": [{
"parts":[
{"file_data":{"mime_type": "'"${MIME_TYPE}"'", "file_uri": "'"${file_uri}"'"}},
{"text": "Caption this image."}]
}]
}' 2> /dev/null > response.json
cat response.json
echo
jq ".candidates[].content.parts[].text" response.json
使用多张图片提示
您可以在单个问题中提供多张图片,方法是在 contents
数组中添加多个图片 Part
对象。这些数据可以是内嵌数据(本地文件或网址)和 File API 引用的混合。
Python
from google import genai
from google.genai import types
client = genai.Client(api_key="GOOGLE_API_KEY")
# Upload the first image
image1_path = "path/to/image1.jpg"
uploaded_file = client.files.upload(file=image1_path)
# Prepare the second image as inline data
image2_path = "path/to/image2.png"
with open(image2_path, 'rb') as f:
img2_bytes = f.read()
# Create the prompt with text and multiple images
response = client.models.generate_content(
model="gemini-2.0-flash",
contents=[
"What is different between these two images?",
uploaded_file, # Use the uploaded file reference
types.Part.from_bytes(
data=img2_bytes,
mime_type='image/png'
)
]
)
print(response.text)
JavaScript
import {
GoogleGenAI,
createUserContent,
createPartFromUri,
} from "@google/genai";
import * as fs from "node:fs";
const ai = new GoogleGenAI({ apiKey: "GOOGLE_API_KEY" });
async function main() {
// Upload the first image
const image1_path = "path/to/image1.jpg";
const uploadedFile = await ai.files.upload({
file: image1_path,
config: { mimeType: "image/jpeg" },
});
// Prepare the second image as inline data
const image2_path = "path/to/image2.png";
const base64Image2File = fs.readFileSync(image2_path, {
encoding: "base64",
});
// Create the prompt with text and multiple images
const response = await ai.models.generateContent({
model: "gemini-2.0-flash",
contents: createUserContent([
"What is different between these two images?",
createPartFromUri(uploadedFile.uri, uploadedFile.mimeType),
{
inlineData: {
mimeType: "image/png",
data: base64Image2File,
},
},
]),
});
console.log(response.text);
}
await main();
Go
// Upload the first image
image1Path := "path/to/image1.jpg"
uploadedFile, _ := client.Files.UploadFromPath(ctx, image1Path, nil)
// Prepare the second image as inline data
image2Path := "path/to/image2.jpeg"
imgBytes, _ := os.ReadFile(image2Path)
parts := []*genai.Part{
genai.NewPartFromText("What is different between these two images?"),
genai.NewPartFromBytes(imgBytes, "image/jpeg"),
genai.NewPartFromURI(uploadedFile.URI, uploadedFile.MIMEType),
}
contents := []*genai.Content{
genai.NewContentFromParts(parts, genai.RoleUser),
}
result, _ := client.Models.GenerateContent(
ctx,
"gemini-2.0-flash",
contents,
nil,
)
fmt.Println(result.Text())
REST
# Upload the first image
IMAGE1_PATH="path/to/image1.jpg"
MIME1_TYPE=$(file -b --mime-type "${IMAGE1_PATH}")
NUM1_BYTES=$(wc -c < "${IMAGE1_PATH}")
DISPLAY_NAME1=IMAGE1
tmp_header_file1=upload-header1.tmp
curl "https://generativelanguage.googleapis.com/upload/v1beta/files?key=${GOOGLE_API_KEY}" \
-D upload-header1.tmp \
-H "X-Goog-Upload-Protocol: resumable" \
-H "X-Goog-Upload-Command: start" \
-H "X-Goog-Upload-Header-Content-Length: ${NUM1_BYTES}" \
-H "X-Goog-Upload-Header-Content-Type: ${MIME1_TYPE}" \
-H "Content-Type: application/json" \
-d "{'file': {'display_name': '${DISPLAY_NAME1}'}}" 2> /dev/null
upload_url1=$(grep -i "x-goog-upload-url: " "${tmp_header_file1}" | cut -d" " -f2 | tr -d "\r")
rm "${tmp_header_file1}"
curl "${upload_url1}" \
-H "Content-Length: ${NUM1_BYTES}" \
-H "X-Goog-Upload-Offset: 0" \
-H "X-Goog-Upload-Command: upload, finalize" \
--data-binary "@${IMAGE1_PATH}" 2> /dev/null > file_info1.json
file1_uri=$(jq ".file.uri" file_info1.json)
echo file1_uri=$file1_uri
# Prepare the second image (inline)
IMAGE2_PATH="path/to/image2.png"
MIME2_TYPE=$(file -b --mime-type "${IMAGE2_PATH}")
if [[ "$(base64 --version 2>&1)" = *"FreeBSD"* ]]; then
B64FLAGS="--input"
else
B64FLAGS="-w0"
fi
IMAGE2_BASE64=$(base64 $B64FLAGS $IMAGE2_PATH)
# Now generate content using both images
curl "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=$GOOGLE_API_KEY" \
-H 'Content-Type: application/json' \
-X POST \
-d '{
"contents": [{
"parts":[
{"text": "What is different between these two images?"},
{"file_data":{"mime_type": "'"${MIME1_TYPE}"'", "file_uri": '$file1_uri'}},
{
"inline_data": {
"mime_type":"'"${MIME2_TYPE}"'",
"data": "'"$IMAGE2_BASE64"'"
}
}
]
}]
}' 2> /dev/null > response.json
cat response.json
echo
jq ".candidates[].content.parts[].text" response.json
对象检测
从 Gemini 2.0 开始,模型会接受进一步训练,以检测图片中的对象并获取其边界框坐标。坐标相对于图片尺寸,比例为 [0, 1000]。您需要根据原始图片大小缩小这些坐标。
Python
from google import genai
from google.genai import types
from PIL import Image
import json
client = genai.Client(api_key="GOOGLE_API_KEY")
prompt = "Detect the all of the prominent items in the image. The box_2d should be [ymin, xmin, ymax, xmax] normalized to 0-1000."
image = Image.open("/path/to/image.png")
config = types.GenerateContentConfig(
response_mime_type="application/json"
)
response = client.models.generate_content(model="gemini-2.0-flash",
contents=[image, prompt],
config=config
)
width, height = image.size
bounding_boxes = json.loads(response.text)
converted_bounding_boxes = []
for bounding_box in bounding_boxes:
abs_y1 = int(bounding_box["box_2d"][0]/1000 * height)
abs_x1 = int(bounding_box["box_2d"][1]/1000 * width)
abs_y2 = int(bounding_box["box_2d"][2]/1000 * height)
abs_x2 = int(bounding_box["box_2d"][3]/1000 * width)
converted_bounding_boxes.append([abs_x1, abs_y1, abs_x2, abs_y2])
print("Image size: ", width, height)
print("Bounding boxes:", converted_bounding_boxes)
如需查看更多示例,请参阅以下食谱:
细分
从 Gemini 2.5 开始,模型不仅可以检测商品,还可以对其进行分割并提供轮廓遮罩。
该模型会预测一个 JSON 列表,其中每个项都代表一个分割掩码。每个项都有一个边界框(“box_2d
”),格式为 [y0, x0, y1, x1]
,归一化坐标介于 0 到 1000 之间,一个用于标识对象的标签(“label
”),最后是边界框内的分割掩码,以 base64 编码的 png 格式表示,即值介于 0 到 255 之间的概率图。需要调整遮罩的大小,使其与边界框的尺寸一致,然后根据置信度阈值进行二值化处理(中点为 127)。
Python
from google import genai
from google.genai import types
from PIL import Image, ImageDraw
import io
import base64
import json
import numpy as np
import os
client = genai.Client()
def parse_json(json_output: str):
# Parsing out the markdown fencing
lines = json_output.splitlines()
for i, line in enumerate(lines):
if line == "```json":
json_output = "\n".join(lines[i+1:]) # Remove everything before "```json"
json_output = json_output.split("```")[0] # Remove everything after the closing "```"
break # Exit the loop once "```json" is found
return json_output
def extract_segmentation_masks(image_path: str, output_dir: str = "segmentation_outputs"):
# Load and resize image
im = Image.open(image_path)
im.thumbnail([1024, 1024], Image.Resampling.LANCZOS)
prompt = """
Give the segmentation masks for the wooden and glass items.
Output a JSON list of segmentation masks where each entry contains the 2D
bounding box in the key "box_2d", the segmentation mask in key "mask", and
the text label in the key "label". Use descriptive labels.
"""
config = types.GenerateContentConfig(
thinking_config=types.ThinkingConfig(thinking_budget=0) # set thinking_budget to 0 for better results in object detection
)
response = client.models.generate_content(
model="gemini-2.5-flash-preview-05-20",
contents=[prompt, im], # Pillow images can be directly passed as inputs (which will be converted by the SDK)
config=config
)
# Parse JSON response
items = json.loads(parse_json(response.text))
# Create output directory
os.makedirs(output_dir, exist_ok=True)
# Process each mask
for i, item in enumerate(items):
# Get bounding box coordinates
box = item["box_2d"]
y0 = int(box[0] / 1000 * im.size[1])
x0 = int(box[1] / 1000 * im.size[0])
y1 = int(box[2] / 1000 * im.size[1])
x1 = int(box[3] / 1000 * im.size[0])
# Skip invalid boxes
if y0 >= y1 or x0 >= x1:
continue
# Process mask
png_str = item["mask"]
if not png_str.startswith("data:image/png;base64,"):
continue
# Remove prefix
png_str = png_str.removeprefix("data:image/png;base64,")
mask_data = base64.b64decode(png_str)
mask = Image.open(io.BytesIO(mask_data))
# Resize mask to match bounding box
mask = mask.resize((x1 - x0, y1 - y0), Image.Resampling.BILINEAR)
# Convert mask to numpy array for processing
mask_array = np.array(mask)
# Create overlay for this mask
overlay = Image.new('RGBA', im.size, (0, 0, 0, 0))
overlay_draw = ImageDraw.Draw(overlay)
# Create overlay for the mask
color = (255, 255, 255, 200)
for y in range(y0, y1):
for x in range(x0, x1):
if mask_array[y - y0, x - x0] > 128: # Threshold for mask
overlay_draw.point((x, y), fill=color)
# Save individual mask and its overlay
mask_filename = f"{item['label']}_{i}_mask.png"
overlay_filename = f"{item['label']}_{i}_overlay.png"
mask.save(os.path.join(output_dir, mask_filename))
# Create and save overlay
composite = Image.alpha_composite(im.convert('RGBA'), overlay)
composite.save(os.path.join(output_dir, overlay_filename))
print(f"Saved mask and overlay for {item['label']} to {output_dir}")
# Example usage
if __name__ == "__main__":
extract_segmentation_masks("path/to/image.png")
如需查看更详细的示例,请参阅食谱指南中的细分示例。

支持的图片格式
Gemini 支持以下图片格式 MIME 类型:
- PNG -
image/png
- JPEG -
image/jpeg
- WEBP -
image/webp
- HEIC -
image/heic
- HEIF -
image/heif
功能
所有 Gemini 模型版本都是多模态的,可用于各种图片处理和计算机视觉任务,包括但不限于图片描述、视觉问答、图片分类、对象检测和分割。
Gemini 可以根据您的质量和性能要求,减少使用专用机器学习模型的需要。
除了通用功能之外,一些较新版本的模型还经过专门训练,可提高专门任务的准确性:
限制和关键技术信息
文件数量限制
Gemini 2.5 Pro/Flash、2.0 Flash、1.5 Pro 和 1.5 Flash 支持每个请求最多 3,600 个图片文件。
令牌计算
- Gemini 1.5 Flash 和 Gemini 1.5 Pro:如果两个尺寸均小于等于 384 像素,则为 258 个令牌。系统会将较大的图片划分为多个图块(每个图块最小 256 像素,最大 768 像素,调整为 768x768),每个图块的费用为 258 个令牌。
- Gemini 2.0 Flash 和 Gemini 2.5 Flash/Pro:如果两个尺寸均小于等于 384 像素,则为 258 个令牌。 系统会将较大的图片划分为 768x768 像素的图块,每个图块的费用为 258 个令牌。
技巧和最佳做法
- 验证图片是否旋转正确。
- 使用清晰、不模糊的图片。
- 使用带文本的单张图片时,请将文本提示放在
contents
数组中的图片部分后面。
后续步骤
本指南介绍了如何上传图片文件,以及如何根据图片输入生成文本输出。如需了解详情,请参阅以下资源: