Midscene Java

midscene-java/
├── packages/
│   ├── core/               # Core module, providing Agent and AI engine
│   ├── web/                # Web automation module
│   │   ├── playwright/     # Playwright implementation
│   │   └── selenium/       # Selenium implementation
│   ├── android/            # Android automation module
│   ├── cli/                # Command line tool
│   ├── examples/           # Example code
│   ├── playground/         # Development testing environment
│   └── tests/              # Test cases
├── apps/                   # Application examples
├── docs/                   # Project documentation and optimization plans
└── wiki/                   # Project wiki documentation

<dependencies>
    <!-- Core module -->
    <dependency>
        <groupId>com.midscene</groupId>
        <artifactId>midscene-core</artifactId>
        <version>0.1.1</version>
    </dependency>
    
    <!-- Web automation modules (choose as needed) -->
    <dependency>
        <groupId>com.midscene</groupId>
        <artifactId>midscene-web-playwright</artifactId>
        <version>0.1.1</version>
    </dependency>
    <dependency>
        <groupId>com.midscene</groupId>
        <artifactId>midscene-web-selenium</artifactId>
        <version>0.1.1</version>
    </dependency>
    
    <!-- Android automation module (choose as needed) -->
    <dependency>
        <groupId>com.midscene</groupId>
        <artifactId>midscene-android</artifactId>
        <version>0.1.1</version>
    </dependency>
</dependencies>

# application.properties
midscene.ai.provider=openai
midscene.ai.model=gpt-4-vision-preview
midscene.ai.api-key=your_openai_api_key_here

package com.example;

import com.midscene.core.Agent;
import com.midscene.web.playwright.PlaywrightPage;
import com.midscene.web.playwright.PlaywrightUIContextProvider;
import com.microsoft.playwright.Playwright;
import com.microsoft.playwright.Browser;
import com.microsoft.playwright.Page;

public class SearchExample {
    public static void main(String[] args) {
        try (Playwright playwright = Playwright.create()) {
            // Create browser instance
            Browser browser = playwright.chromium().launch();
            Page page = browser.newPage();
            
            // Create PlaywrightPage wrapper
            PlaywrightPage playwrightPage = new PlaywrightPage(page);
            
            // Create Agent
            Agent agent = new Agent(new PlaywrightUIContextProvider(playwrightPage));
            
            // Navigate to website
            page.navigate("https://www.baidu.com");
            
            // Use natural language for search
            agent.aiAction("Type 'Java tutorial' in the search box");
            agent.aiAction("Click the search button");
            
            // Verify search results
            agent.aiAssert("The page displays search results for Java tutorials");
            
            System.out.println("✅ Search operation completed!");
            
            // Close browser
            browser.close();
        }
    }
}

package com.example;

import com.midscene.core.Agent;
import com.midscene.web.playwright.PlaywrightPage;
import com.midscene.web.playwright.PlaywrightUIContextProvider;
import com.microsoft.playwright.Playwright;
import com.microsoft.playwright.Browser;
import com.microsoft.playwright.Page;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class ExtractExample {
    public static void main(String[] args) {
        try (Playwright playwright = Playwright.create()) {
            Browser browser = playwright.chromium().launch();
            Page page = browser.newPage();
            
            PlaywrightPage playwrightPage = new PlaywrightPage(page);
            Agent agent = new Agent(new PlaywrightUIContextProvider(playwrightPage));
            
            // Visit news website
            page.navigate("https://news.example.com");
            
            // Extract structured data
            Map<String, Object> schema = new HashMap<>();
            schema.put("articles", List.of(
                Map.of(
                    "title", "News title",
                    "time", "Publish time",
                    "summary", "News summary"
                )
            ));
            
            Map<String, Object> newsData = agent.aiExtract(schema);
            
            // Output results
            List<Map<String, String>> articles = (List<Map<String, String>>) newsData.get("articles");
            for (Map<String, String> article : articles) {
                System.out.println("📰 " + article.get("title"));
                System.out.println("⏰ " + article.get("time"));
                System.out.println("📄 " + article.get("summary") + "\n");
            }
            
            browser.close();
        }
    }
}

package com.example;

import com.midscene.core.Agent;
import com.midscene.android.AndroidDevice;
import com.midscene.android.AndroidUIContextProvider;

import java.util.concurrent.CompletableFuture;

public class AndroidExample {
    public static void main(String[] args) {
        // Connect to Android device
        AndroidDevice device = new AndroidDevice();
        CompletableFuture<Void> connectFuture = device.connect();
        connectFuture.join(); // Wait for connection to complete
        
        try {
            // Create Agent
            Agent agent = new Agent(new AndroidUIContextProvider(device));
            
            // Launch application
            agent.aiAction("Launch the settings app");
            
            // Perform operations
            agent.aiAction("Tap on the Wi-Fi option");
            agent.aiAssert("The Wi-Fi settings page is open");
            
            System.out.println("✅ Android automation operation completed!");
        } finally {
            device.disconnect();
        }
    }
}

# Clone the repository
git clone https://github.com/Master-Frank/midscene-java.git
cd midscene-java

# Build the project
mvn clean install

# Run tests
mvn test