Google Apps Script で Gemini API を使う

const apiKey = PropertiesService.getScriptProperties().getProperty("GEMINI_API_KEY");
const gemini = 'gemini-2.5-flash';
const geminiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${gemini}:generateContent?key=${apiKey}`;

function callGemini(parts, schema){
  const payload = {
    contents: [ { parts }  ],
    generationConfig: {
      response_mime_type: "application/json",
      response_json_schema: schema
    }
  };

  const options = {
    method: 'post',
    contentType: `application/json`,
    payload: JSON.stringify(payload)
  };

  const response = UrlFetchApp.fetch(geminiUrl, options);
  const data = JSON.parse(response);
  return data.candidates[0];
}

const apiKey = PropertiesService.getScriptProperties().getProperty("GEMINI_API_KEY");
const gemma = 'gemma-3-4b-it';
const gemmaUrl = `https://generativelanguage.googleapis.com/v1beta/models/${gemma}:generateContent?key=${apiKey}`;const geminiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${gemini}:generateContent?key=${apiKey}`;

function callGemma(parts){
  const payload = {
    contents: [ { parts }  ],
  };

  const options = {
    method: 'post',
    contentType: `application/json`,
    payload: JSON.stringify(payload)
  };

  const response = UrlFetchApp.fetch(gemmaUrl, options);
  const data = JSON.parse(response);
  return data.candidates[0];
}

function testDescribeWord(){
  describeWord("恵方巻");
}

function describeWord(word){
  const text = `
You are a person who is very knowledgeable about Japanese culture.
You are asked to explain the word ${word}.

Explain the word and Japanese culture through it, in a friendly, conversational tone.

Guidelines:
- Use natural, spoken English
- Explain it in ONE or TWO short sentences.
- Keep it casual and friendly, like talking to a friend
- Do not sound like a textbook or tour guide
- Avoid overly technical terms
- If there is no Japanese aspect, say so honestly instead of forcing an explanation
`

  const parts = [{ text: text }];

  const result = callGemma(parts);
  // const result = callGemini(parts);

  console.log(result.content);
  return result.content;
}

function getImageAsBase64(fileId) {
  try {
    const file = DriveApp.getFileById(fileId);
    const imageBlob = file.getBlob();
    const imageBytes = imageBlob.getBytes();
    const data = Utilities.base64Encode(imageBytes);
    const mimeType = imageBlob.getContentType();
    return { mime_type: mimeType, data };
  } catch (e) {
    console.error("Error reading image from Google Drive: " + e.toString());
    return null;
  }
}

const imageFileId = 'ここに画像ファイルのIDを入力してください';

function testDescribeImageGemma(){
  const image = getImageAsBase64(imageFileId);
  describeImageGemma(image);
}

const prompt = `
You are a person who is very knowledgeable about Japanese culture.
You are going out together with the user, casually walking around and looking at things through the camera.

When an image is provided, look for elements related to Japanese culture in the image
(such as buildings, streets, signs, food, clothing, religious objects, seasonal details, or everyday customs).

Explain what you notice in a friendly, conversational tone, as if you are saying:
“Hey, look at that,” or “See that over there?”

Guidelines:
- Use natural, spoken English
- Explain it in ONE or TWO short sentences.
- Keep it casual and friendly, like talking to a friend
- Do not sound like a textbook or tour guide
- Avoid overly technical terms
- If you are not completely sure, say things like “It looks like…” or “This might be…”
- If there are no clear Japanese cultural elements, say so honestly instead of forcing an explanation

Your explanation should feel like:
A culturally knowledgeable companion pointing something out during a walk.`

function describeImageGemma(image){
  const parts = [
    { inline_data: image },
    { text: prompt }
  ];

  const result = callGemma(parts);
  console.log(result.content);
  return result.content;
}

function testDescribeImageGemini(){
  const image = getImageAsBase64(imageFileId);
  describeImageGemini(image);
}

function describeImageGemini(image){
  const parts = [
    { inline_data: image },
    { text: prompt }
  ];

  const schema = {
    type: "object",
    properties: {
      elements: {
        type: "array",
        description: "Elements found in the picture with its bounding-box.",
        items: {
          type: "object",
          properties: {
            name: { type: "string" },
            explanation: { type: "string" },
            x: { type: "number", description: "Normalized x-coordinate of the top-left corner (0.0 to 1.0)." },
            y: { type: "number", description: "Normalized y-coordinate of the top-left corner (0.0 to 1.0)." },
            w: { type: "number", description: "Normalized width of the bounding box (0.0 to 1.0)." },
            h: { type: "number", description: "Normalized height of the bounding box (0.0 to 1.0)." }
          },
          required: ["name", "explanation", "x", "y", "w", "h"]
        }
      }
    }
  }

  const result = callGemini(parts, schema);
  console.log(result.content);
  return result.content;
}

function doGet(){
  const template = HtmlService.createTemplateFromFile("app");
  return template.evaluate().addMetaTag('viewport', 'width=device-width, initial-scale=1');
}

<!DOCTYPE html>
<html>
  <head>
    <base target="_top">
    <style>
      body { font-family: sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; text-align: center; }
      #video { display: block; width: 100%; height: auto; max-width: 640px; }
      #cameraWrapper { position: relative; display: block; margin: 0 auto; background: #000; border-radius: 8px; overflow: hidden; width: 100%; max-width: 640px; }
      #overlayCanvas { position: absolute; top: 0; left: 0; width: 100%; height: 100%; pointer-events: none; }
      button { padding: 10px 20px; margin: 5px; cursor: pointer; border-radius: 5px; border: none; background: #007bff; color: white; }
      button:hover { background: #0056b3; }
      #wordInput { padding: 10px; width: 60%; }
      #status { display: block; margin: 10px 0; color: #666; font-style: italic; }
    </style>
  </head>
  <body>
    <h1>Describe Japanese Culture</h1>
    <div>
      <input id="wordInput" type="text" placeholder="例:おしぼり,もったいない…"></input>
      <button id="askWordButton">Ask</button>
    </div>
    <div id="cameraWrapper">
      <video id="video" autoplay playsinline></video>
      <canvas id="overlayCanvas"></canvas>
    </div>
    <canvas id="captureCanvas" style="display: none"></canvas>
    <div>
      <button id="switchButton">Switch Camera</button>
      <button id="startButton">Start Camera</button>
      <button id="geminiButton">Ask Gemini</button>
      <button id="gemmaButton">Ask Gemmma</button>
    </div>
    <div>
       <span id="status">Status: start</span>
    </div>

    <script>
      const status = document.getElementById("status");
      const wordInput = document.getElementById("wordInput");
      const video = document.getElementById("video");
      const canvas = document.getElementById("captureCanvas");
      const overlayCanvas = document.getElementById("overlayCanvas");
      const askWordButton = document.getElementById("askWordButton");
      const switchButton = document.getElementById("switchButton");
      const startButton = document.getElementById("startButton");
      const gemmaButton = document.getElementById("gemmaButton");
      const geminiButton = document.getElementById("geminiButton");

      askWordButton.addEventListener("click", askWord);
      switchButton.addEventListener("click", switchCamera);
      startButton.addEventListener("click", startCamera);
      gemmaButton.addEventListener("click", askImageGemma);
      geminiButton.addEventListener("click", askImageGemini);

////////////////////////////////////////////////////////////////////////////////////////////////////
// AI usage with text only propmt
////////////////////////////////////////////////////////////////////////////////////////////////////

      function askWord(){
        const word = wordInput.value;
        if(!word){
          status.textContent = "Please input any word.";
          return;
        }
        status.textContent = "Waiting for an explanation of word: " + word;
        google.script.run.withSuccessHandler(explanationReceived).describeWord(word);
      }

      function explanationReceived(data){
        console.log(data);
        const text = data.parts[0].text;
        status.textContent = "Explanation received: " + text;
        
        window.speechSynthesis.cancel();
        const utterance = new SpeechSynthesisUtterance(text);
        utterance.lang = "en-US";
        utterance.rate = 1.1;
        window.speechSynthesis.speak(utterance);
      }

////////////////////////////////////////////////////////////////////////////////////////////////////
// AI usage with image data
////////////////////////////////////////////////////////////////////////////////////////////////////
      let currentFacingMode = "environment";

      async function switchCamera() {
        currentFacingMode = (currentFacingMode === "user") ? "environment" : "user"; 
        if (video.srcObject) {
          video.srcObject.getTracks().forEach(track => track.stop());
        }
        startCamera();
      }

      async function startCamera(){
        try {
          const constraints = { 
            video: { facingMode: currentFacingMode } 
          };
          const stream = await navigator.mediaDevices.getUserMedia(constraints);
          video.srcObject = stream;
          
          video.onloadedmetadata = () => {
            overlayCanvas.width = video.videoWidth;
            overlayCanvas.height = video.videoHeight;
            canvas.width = video.videoWidth;
            canvas.height = video.videoHeight;
            status.textContent = `Camera: ${currentFacingMode} (${video.videoWidth}x${video.videoHeight})`;
          };
        } catch(err){
          console.error(err);
          status.textContent = "Camera Error: " + err.message;
        }
      }

      function askImageGemma(){
        const data = getImageData();
        google.script.run.withSuccessHandler(explanationReceived).describeImageGemma({ data, mime_type: "image/jpeg" });
        status.textContent = "Waiting for explanation...";
      }

      function askImageGemini(){
        const data = getImageData();
        google.script.run.withSuccessHandler(geminiReceived).describeImageGemini({ data, mime_type: "image/jpeg" });
        status.textContent = "Waiting for detailed explanation using Gemini...";
      }

      function getImageData(){
        const context = canvas.getContext("2d");
        context.drawImage(video, 0, 0, canvas.width, canvas.height);
        return canvas.toDataURL("image/jpeg", 0.5).split(",")[1];
      }

      function geminiReceived(data){
        status.textContent = "Gemini result received.";
        console.log(data);
        const text = data.parts[0].text;
        const elements = JSON.parse(text).elements;
        speakElements(elements);
      }

////////////////////////////////////////////////////////////////////////////////////////////////////
// Speak and show elements one-by-one
////////////////////////////////////////////////////////////////////////////////////////////////////

      let speechQueue = [];
      let currentIndex = 0;
      let isSpeaking = false;

      function speakElements(elements){
        window.speechSynthesis.cancel();
        speechQueue = elements;
        currentIndex = 0;
        isSpeaking = false;

        speakNext();
      }

      // Speak/show an element and issue the next task
      function speakNext() {
        if (currentIndex >= speechQueue.length) {
          isSpeaking = false;
          highlightElement(null);
          return;
        }

        const element = speechQueue[currentIndex];
        isSpeaking = true;

        highlightElement(element);

        const utterance = new SpeechSynthesisUtterance(element.explanation);
        utterance.lang = "en-US";
        utterance.rate = 1.1;

        utterance.onend = () => {
          currentIndex++;
          speakNext();
        };

        window.speechSynthesis.speak(utterance);
      }

      // Show an element
      function highlightElement(e) {
        if(!e) return;
        const overlayCtx = prepareCanvasContext();

        // Convert normalized coordinates (0.0 to 1.0) to pixel values
        const canvasW = overlayCanvas.width;
        const canvasH = overlayCanvas.height;
        const x = e.x * canvasW;
        const y = e.y * canvasH;
        const w = e.w * canvasW;
        const h = e.h * canvasH;

        // Draw bounding-box rectangle
        overlayCtx.strokeStyle = "rgba(255, 0, 0, 0.8)";
        overlayCtx.lineWidth = 2;
        overlayCtx.strokeRect(x, y, w, h);

        // Draw label
        const padding = 6;
        overlayCtx.font = "14px sans-serif";
        const text = e.name || e.explanation;
        const textWidth = overlayCtx.measureText(text).width;
        overlayCtx.fillStyle = "rgba(0, 0, 0, 0.6)";
        overlayCtx.fillRect(x, y - 22, textWidth + padding * 2, 20);
        overlayCtx.fillStyle = "white";
        overlayCtx.fillText(text, x + padding, y - 7);
      }

      function prepareCanvasContext(){
        const overlayCtx = overlayCanvas.getContext("2d");
        const rect = overlayCanvas.getBoundingClientRect();
  
        if (overlayCanvas.width !== rect.width || overlayCanvas.height !== rect.height) {
          overlayCanvas.width = rect.width;
          overlayCanvas.height = rect.height;
        }

        overlayCtx.clearRect(0, 0, overlayCanvas.width, overlayCanvas.height);
        return overlayCtx;
      }
    </script>
  </body>
</html>

function askWord(){
    const word = wordInput.value;
    if(!word){
        status.textContent = "Please input any word.";
        return;
    }
    status.textContent = "Waiting for an explanation of word: " + word;
    google.script.run.withSuccessHandler(explanationReceived).describeWord(word);
}

function explanationReceived(data){
    console.log(data);
    const text = data.parts[0].text;
    status.textContent = "Explanation received: " + text;

    window.speechSynthesis.cancel();
    const utterance = new SpeechSynthesisUtterance(text);
    utterance.lang = "en-US";
    utterance.rate = 1.1;
    window.speechSynthesis.speak(utterance);
}

function getImageData(){
    const context = canvas.getContext("2d");
    context.drawImage(video, 0, 0, canvas.width, canvas.height);
    return canvas.toDataURL("image/jpeg", 0.5).split(",")[1];
}