From b7f8254fec64889114d4c42553e23d5288c3f84f Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Sat, 28 Feb 2026 11:22:04 +0800 Subject: [PATCH] feat: add @midscene/harmony demo examples Add HarmonyOS device automation demo examples using @midscene/harmony: - JavaScript SDK demo: eBay search with aiAct, aiQuery, aiAssert, etc. - Vitest demo: todo list test and settings page scroll test --- harmony/javascript-sdk-demo/.gitignore | 2 + harmony/javascript-sdk-demo/README.md | 37 +++++++++++ harmony/javascript-sdk-demo/demo.ts | 57 ++++++++++++++++ harmony/javascript-sdk-demo/package.json | 18 +++++ harmony/vitest-demo/.gitignore | 2 + harmony/vitest-demo/README.md | 45 +++++++++++++ harmony/vitest-demo/package.json | 18 +++++ harmony/vitest-demo/tests/setting.test.ts | 27 ++++++++ harmony/vitest-demo/tests/todo.test.ts | 81 +++++++++++++++++++++++ 9 files changed, 287 insertions(+) create mode 100644 harmony/javascript-sdk-demo/.gitignore create mode 100644 harmony/javascript-sdk-demo/README.md create mode 100644 harmony/javascript-sdk-demo/demo.ts create mode 100644 harmony/javascript-sdk-demo/package.json create mode 100644 harmony/vitest-demo/.gitignore create mode 100644 harmony/vitest-demo/README.md create mode 100644 harmony/vitest-demo/package.json create mode 100644 harmony/vitest-demo/tests/setting.test.ts create mode 100644 harmony/vitest-demo/tests/todo.test.ts diff --git a/harmony/javascript-sdk-demo/.gitignore b/harmony/javascript-sdk-demo/.gitignore new file mode 100644 index 0000000..3012a4b --- /dev/null +++ b/harmony/javascript-sdk-demo/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +midscene_run/ diff --git a/harmony/javascript-sdk-demo/README.md b/harmony/javascript-sdk-demo/README.md new file mode 100644 index 0000000..b484798 --- /dev/null +++ b/harmony/javascript-sdk-demo/README.md @@ -0,0 +1,37 @@ +# HarmonyOS demo (JavaScript SDK) + +This is a demo to show how to use `@midscene/harmony` JavaScript SDK to control HarmonyOS devices. + +## Steps + +### Preparation + +Create `.env` file + +```shell +# Replace with your own API key +MIDSCENE_MODEL_BASE_URL="https://.../compatible-mode/v1" +MIDSCENE_MODEL_API_KEY="sk-abcdefghijklmnopqrstuvwxyz" +MIDSCENE_MODEL_NAME="qwen3-vl-plus" +MIDSCENE_MODEL_FAMILY="qwen3-vl" +``` + +Connect a HarmonyOS device with [HDC](https://developer.huawei.com/consumer/en/doc/harmonyos-guides-V5/ide-hdc-V5) + +Refer to this document if you want to use other models like Qwen: https://midscenejs.com/model-strategy.html + +### Install + +```bash +npm install +``` + +### Run + +```bash +npm test +``` + +## Reference + +https://midscenejs.com/harmony-api-reference diff --git a/harmony/javascript-sdk-demo/demo.ts b/harmony/javascript-sdk-demo/demo.ts new file mode 100644 index 0000000..753bd27 --- /dev/null +++ b/harmony/javascript-sdk-demo/demo.ts @@ -0,0 +1,57 @@ +import { + HarmonyAgent, + HarmonyDevice, + getConnectedDevices, +} from '@midscene/harmony'; +import 'dotenv/config'; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +Promise.resolve( + (async () => { + const devices = await getConnectedDevices(); + const device = new HarmonyDevice(devices[0].deviceId); + + // 👀 init Midscene agent + const agent = new HarmonyAgent(device); + await device.connect(); + await agent.aiAct('open Browser and go to https://www.ebay.com'); + + await sleep(5000); + + // 👀 type keywords, perform a search + await agent.aiAct('type "Headphones" in search box, click search button'); + + // 👀 wait for the loading + await agent.aiWaitFor('there is at least one headphone item on page'); + + // 👀 understand the page content, find the items + const items = await agent.aiQuery( + '{itemTitle: string, price: Number}[], find item in list and corresponding price', + ); + console.log('headphones in stock', items); + + const isMoreThan1000 = await agent.aiBoolean( + 'Is the price of the headphones more than 1000?', + ); + console.log('isMoreThan1000', isMoreThan1000); + + const price = await agent.aiNumber( + 'What is the price of the first headphone?', + ); + console.log('price', price); + + const name = await agent.aiString( + 'What is the name of the first headphone?', + ); + console.log('name', name); + + const location = await agent.aiLocate( + 'What is the location of the first headphone?', + ); + console.log('location', location); + + // 👀 assert by AI + await agent.aiAssert('There is a category filter on the left'); + })(), +); diff --git a/harmony/javascript-sdk-demo/package.json b/harmony/javascript-sdk-demo/package.json new file mode 100644 index 0000000..93fa924 --- /dev/null +++ b/harmony/javascript-sdk-demo/package.json @@ -0,0 +1,18 @@ +{ + "name": "harmony-demo", + "private": true, + "version": "1.0.0", + "description": "> quick start", + "main": "index.js", + "type": "module", + "scripts": { + "test": "tsx demo.ts" + }, + "author": "", + "license": "MIT", + "devDependencies": { + "@midscene/harmony": "latest", + "dotenv": "^16.4.5", + "tsx": "4.20.1" + } +} \ No newline at end of file diff --git a/harmony/vitest-demo/.gitignore b/harmony/vitest-demo/.gitignore new file mode 100644 index 0000000..3012a4b --- /dev/null +++ b/harmony/vitest-demo/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +midscene_run/ diff --git a/harmony/vitest-demo/README.md b/harmony/vitest-demo/README.md new file mode 100644 index 0000000..a575aec --- /dev/null +++ b/harmony/vitest-demo/README.md @@ -0,0 +1,45 @@ +# HarmonyOS demo (vitest) + +This is a demo to show how to use HDC to control HarmonyOS devices for automation tasks. + +## Steps + +### Preparation + +Create `.env` file + +```shell +# Replace with your own API key +MIDSCENE_MODEL_BASE_URL="https://.../compatible-mode/v1" +MIDSCENE_MODEL_API_KEY="sk-abcdefghijklmnopqrstuvwxyz" +MIDSCENE_MODEL_NAME="qwen3-vl-plus" +MIDSCENE_MODEL_FAMILY="qwen3-vl" +``` + +Connect a HarmonyOS device with [HDC](https://developer.huawei.com/consumer/en/doc/harmonyos-guides-V5/ide-hdc-V5) + +Refer to this document if you want to use other models like Qwen: https://midscenejs.com/model-strategy.html + +### Install + +```bash +npm install +``` + +### Run + +case1: Settings page scroll demo + +```bash +npm run test -- setting.test.ts +``` + +case2: Todo app demo + +```bash +npm run test -- todo.test.ts +``` + +## Reference + +https://midscenejs.com/harmony-api-reference diff --git a/harmony/vitest-demo/package.json b/harmony/vitest-demo/package.json new file mode 100644 index 0000000..d7e804a --- /dev/null +++ b/harmony/vitest-demo/package.json @@ -0,0 +1,18 @@ +{ + "name": "harmony-with-vitest-demo", + "private": true, + "version": "1.0.0", + "main": "index.js", + "type": "module", + "scripts": { + "test": "vitest --run" + }, + "author": "", + "license": "MIT", + "devDependencies": { + "@midscene/harmony": "latest", + "@types/node": "^18.0.0", + "dotenv": "^16.4.5", + "vitest": "^2.1.8" + } +} \ No newline at end of file diff --git a/harmony/vitest-demo/tests/setting.test.ts b/harmony/vitest-demo/tests/setting.test.ts new file mode 100644 index 0000000..f5acad5 --- /dev/null +++ b/harmony/vitest-demo/tests/setting.test.ts @@ -0,0 +1,27 @@ +import { agentFromHdcDevice, getConnectedDevices } from '@midscene/harmony'; +import { describe, it, vi } from 'vitest'; +import 'dotenv/config'; + +vi.setConfig({ + testTimeout: 90 * 1000, +}); + +describe( + 'harmony integration', + async () => { + await it('HarmonyOS settings page demo for scroll', async () => { + const devices = await getConnectedDevices(); + const agent = await agentFromHdcDevice(devices[0].deviceId); + + await agent.launch('Settings'); + + await agent.aiAct('scroll list to bottom'); + await agent.aiAct('open "Bluetooth" settings'); + await agent.aiAct('scroll list to bottom'); + await agent.aiAct('scroll list to top'); + await agent.aiAct('swipe down one screen'); + await agent.aiAct('swipe up one screen'); + }); + }, + 360 * 1000, +); diff --git a/harmony/vitest-demo/tests/todo.test.ts b/harmony/vitest-demo/tests/todo.test.ts new file mode 100644 index 0000000..a75e906 --- /dev/null +++ b/harmony/vitest-demo/tests/todo.test.ts @@ -0,0 +1,81 @@ +import { + HarmonyAgent, + HarmonyDevice, + getConnectedDevices, +} from '@midscene/harmony'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import 'dotenv/config'; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +vi.setConfig({ + testTimeout: 240 * 1000, + hookTimeout: 240 * 1000, +}); + +const pageUrl = 'https://todomvc.com/examples/react/dist/'; + +describe('Test todo list', () => { + let agent: HarmonyAgent; + + beforeAll(async () => { + const devices = await getConnectedDevices(); + const device = new HarmonyDevice(devices[0].deviceId); + agent = new HarmonyAgent(device); + await device.connect(); + await agent.aiAct(`open Browser and go to ${pageUrl}`); + await sleep(3000); + }); + + it( + 'ai todo', + async () => { + await agent.aiAct( + "type 'Study JS today' in the task box input and press the Enter key", + ); + await agent.aiAct( + "type 'Study Rust tomorrow' in the task box input and press the Enter key", + ); + await agent.aiAct( + "type 'Study AI the day after tomorrow' in the task box input and press the Enter key", + ); + await agent.aiAct( + 'move the mouse to the second item in the task list and click the delete button on the right of the second task', + ); + await agent.aiAct( + 'click the check button on the left of the second task', + ); + await agent.aiAct( + "click the 'completed' status button below the task list", + ); + + const list = await agent.aiQuery('string[], the complete task list'); + expect(list.length).toEqual(1); + + await agent.aiAssert( + 'Near the bottom of the list, there is a tip shows "1 item left".', + ); + + const name = await agent.aiString( + 'What is the name of the first todo?', + ); + console.log('name', name); + + const todoCount = await agent.aiNumber( + 'How many todos are there in the list?', + ); + console.log('todoCount', todoCount); + + const isAllCompleted = await agent.aiBoolean( + 'Is all todos completed?', + ); + console.log('isAllCompleted', isAllCompleted); + + const location = await agent.aiLocate( + 'What is the location of the first todo?', + ); + console.log('location', location); + }, + 720 * 1000, + ); +});