Web Browser option for Gemini AI API

Gemini API
,
1

I want to build an easy-to-get version of RAG that will fetch the files from urls populated in my google sheet that serves as a GUI, but the Gemini AI API models that i connected from Google AI Studio endpoints don’t seem to have the direct access to external file links through my urls.

What are the easiest steps to make it happen with an AI model without extracting the data from PDFs (because the unsctructured data is almost impossible to extract precisely since there are a lot of scanned photos, images and tables, that should be taken into consideration).
Creating a vector database is a very time-consuming option, therefore the best way would be to recreate something like notebooklm or google ai studio UI, but in my own GUI through the certain APIs.

Here is the example of my current apps script code:

function gemini() {
  // Sheet and cell configuration
  var spreadsheetId = "1SkKeNkHHD1LnHVtg7-L553i3J3PoF7fLy2fMECD3Pv4"; // Replace with your actual ID
  var sheetName = "Sheet1"; // Change this to your sheet's name
  var sheet = SpreadsheetApp.openById(spreadsheetId).getSheetByName(sheetName);

  // Define the cells for Links, Query, and Answer
  var links = sheet.getRange("B4").getValue(); // Cell B4 (Links)
  var query = sheet.getRange("B5").getValue(); // Cell B5 (Query)
  var answerCell = sheet.getRange("B6"); // Cell B6 (Answer)

  // Gemini API Configuration
  var apiKey = "AIzaSyCXHbt90WS_OU665wAYz6ss9pBXtCgcZdU"; // Replace with your actual API key
  var modelName = "gemini-2.0-flash-exp"; // Using the model from your current script
  var geminiApiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:generateContent?key=${apiKey}`;

  // Validate if links and query are provided
  if (!links || !query) {
    Logger.log("Missing links or query.");
    answerCell.setValue("Error: Missing links or query.");
    return;
  }

  // Generate prompt for Gemini
  var prompt = `Please answer the following question: ${query}`;

  // Call Gemini API
  var answer = callGeminiApi(prompt, links, query, geminiApiUrl); // Pass links and query

  // Update the answer cell in the sheet
  if (answer) {
    answerCell.setValue(answer);
  } else {
    answerCell.setValue("Error: No response received.");
  }
}

function callGeminiApi(prompt, links, query, geminiApiUrl) {
    try {
        // Combine system instructions and user prompt
        var systemInstructions = `You are a specialized assistant in solar power systems. You have access to the google cloud storage and you're provided with URLs of google cloud storage files in ${links}. Please examine the content of the files relevant to the question: ${query}. Interpret and extract details from technical documents accurately. Provide answers concisely and specify when context is insufficient.`;
        var fullPrompt = systemInstructions + "\n\n" + prompt;

        // Correct Payload structure using 'contents'
        var payload = {
            "contents": [
                {
                    "parts": [
                        {
                            "text": fullPrompt
                        }
                    ]
                }
            ]
        };

        var options = {
            "method": "post",
            "contentType": "application/json",
            "payload": JSON.stringify(payload)
        };

        Logger.log("Payload being sent: " + JSON.stringify(payload)); // Log the payload

        var response = UrlFetchApp.fetch(geminiApiUrl, options);
        var json = JSON.parse(response.getContentText());

        // Extract the answer
        if (json && json.candidates && json.candidates[0] && json.candidates[0].content && json.candidates[0].content.parts && json.candidates[0].content.parts[0] && json.candidates[0].content.parts[0].text) {
            return json.candidates[0].content.parts[0].text;
        } else {
            Logger.log("No valid output found in Gemini response.");
            return "No valid output found.";
        }
    } catch (e) {
        Logger.log("Error calling Gemini API: " + e.toString());
        return "Error: " + e.message;
    }
}