Skip to content

Commit 55eeb08

Browse files
KiteniteKitenite
authored
Web version pt.6 (#1767)
* Add build script * Add iframe polyfill * Add preload apis * Established connection * More iframe connection * Connect with mousedown * Better port management * Fix frame * Fix next dep * Update render
1 parent 8b5305c commit 55eeb08

File tree

38 files changed

+2346
-164
lines changed

38 files changed

+2346
-164
lines changed

‎apps/web/client/src/app/project/[id]/_components/canvas/frame/gesture.tsx

Lines changed: 69 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,85 +1,82 @@
11
import { useEditorEngine } from '@/components/store';
22
import { getRelativeMousePositionToWebview } from '@/components/store/editor/engine/overlay/utils';
3-
import { EditorMode, type MouseAction } from '@onlook/models/editor';
4-
import type { ElementPosition } from '@onlook/models/element';
3+
import { EditorMode, MouseAction } from '@onlook/models/editor';
4+
import type { DomElement, ElementPosition } from '@onlook/models/element';
5+
import type { WebFrame } from '@onlook/models/frame';
56
import { cn } from '@onlook/ui/utils';
67
import throttle from 'lodash/throttle';
78
import { observer } from 'mobx-react-lite';
89
import { useCallback, useEffect, useMemo } from 'react';
910
import { RightClickMenu } from './right-click';
10-
11-
export const GestureScreen = observer(() => {
11+
import type { WebFrameView } from './web-frame';
12+
export const GestureScreen = observer(({ frame, webFrameRef }: { frame: WebFrame, webFrameRef: React.RefObject<WebFrameView | null> }) => {
1213
const editorEngine = useEditorEngine();
1314
const isResizing = false;
15+
const webFrame = webFrameRef.current;
1416

15-
const getWebview = useCallback((): Electron.WebviewTag => {
16-
// const webview = webviewRef.current as Electron.WebviewTag | null;
17-
// if (!webview) {
18-
// throw Error('No webview found');
19-
// }
20-
// return webview;
21-
}, []);
17+
if (!webFrame) {
18+
console.log('No web frame found in gesture screen for frame', frame.id);
19+
return null;
20+
}
2221

23-
const getRelativeMousePosition = useCallback(
24-
(e: React.MouseEvent<HTMLDivElement>): ElementPosition => {
25-
const webview = getWebview();
26-
return getRelativeMousePositionToWebview(e, webview);
27-
},
28-
[getWebview],
29-
);
22+
const getRelativeMousePosition = (e: React.MouseEvent<HTMLDivElement>): ElementPosition => {
23+
return getRelativeMousePositionToWebview(e, webFrame);
24+
}
3025

3126
const handleMouseEvent = useCallback(
3227
async (e: React.MouseEvent<HTMLDivElement>, action: MouseAction) => {
33-
// const webview = getWebview();
34-
// const pos = getRelativeMousePosition(e);
35-
// const el: DomElement = await webview.executeJavaScript(
36-
// `window.api?.getElementAtLoc(${pos.x}, ${pos.y}, ${action === MouseAction.MOUSE_DOWN || action === MouseAction.DOUBLE_CLICK})`,
37-
// );
38-
// if (!el) {
39-
// return;
40-
// }
41-
42-
// switch (action) {
43-
// case MouseAction.MOVE:
44-
// editorEngine.elements.mouseover(el, webview);
45-
// if (e.altKey) {
46-
// editorEngine.elements.showMeasurement();
47-
// } else {
48-
// editorEngine.overlay.removeMeasurement();
49-
// }
50-
// break;
51-
// case MouseAction.MOUSE_DOWN:
52-
// if (el.tagName.toLocaleLowerCase() === 'body') {
53-
// editorEngine.webview.select(webview);
54-
// return;
55-
// }
56-
// // Ignore right-clicks
57-
// if (e.button == 2) {
58-
// break;
59-
// }
60-
// if (editorEngine.text.isEditing) {
61-
// editorEngine.text.end();
62-
// }
63-
// if (e.shiftKey) {
64-
// editorEngine.elements.shiftClick(el, webview);
65-
// } else {
66-
// editorEngine.move.start(el, pos, webview);
67-
// editorEngine.elements.click([el], webview);
68-
// }
69-
// break;
70-
// case MouseAction.DOUBLE_CLICK:
71-
// editorEngine.text.start(el, webview);
72-
// break;
73-
// }
28+
const pos = getRelativeMousePosition(e);
29+
const shouldGetStyle = [MouseAction.MOUSE_DOWN, MouseAction.DOUBLE_CLICK].includes(action);
30+
const el: DomElement = await webFrame.getElementAtLoc(pos.x, pos.y, shouldGetStyle);
31+
if (!el) {
32+
console.log('No element found');
33+
return;
34+
}
35+
36+
switch (action) {
37+
case MouseAction.MOVE:
38+
// editorEngine.elements.mouseover(el, webview);
39+
// if (e.altKey) {
40+
// editorEngine.elements.showMeasurement();
41+
// } else {
42+
// editorEngine.overlay.removeMeasurement();
43+
// }
44+
break;
45+
case MouseAction.MOUSE_DOWN:
46+
console.log('mouse down', el);
47+
// if (el.tagName.toLocaleLowerCase() === 'body') {
48+
// editorEngine.webview.select(webview);
49+
// return;
50+
// }
51+
// // Ignore right-clicks
52+
// if (e.button == 2) {
53+
// break;
54+
// }
55+
// if (editorEngine.text.isEditing) {
56+
// editorEngine.text.end();
57+
// }
58+
// if (e.shiftKey) {
59+
// editorEngine.elements.shiftClick(el, webview);
60+
// } else {
61+
// editorEngine.move.start(el, pos, webview);
62+
// editorEngine.elements.click([el], webview);
63+
// }
64+
break;
65+
case MouseAction.DOUBLE_CLICK:
66+
// editorEngine.text.start(el, webview);
67+
break;
68+
}
7469
},
75-
[getWebview, getRelativeMousePosition, editorEngine],
70+
[getRelativeMousePosition, editorEngine],
7671
);
7772

7873
const throttledMouseMove = useMemo(
7974
() =>
8075
throttle((e: React.MouseEvent<HTMLDivElement>) => {
81-
// if (editorEngine.state.move.isDragging) {
82-
// editorEngine.state.move.drag(e, getRelativeMousePosition);
76+
handleMouseEvent(e, MouseAction.MOVE);
77+
78+
// if (editorEngine.move.isDragging) {
79+
// editorEngine.move.drag(e, getRelativeMousePosition);
8380
// } else if (
8481
// editorEngine.state.editorMode === EditorMode.DESIGN ||
8582
// ((editorEngine.state.editorMode === EditorMode.INSERT_DIV ||
@@ -107,7 +104,7 @@ export const GestureScreen = observer(() => {
107104
// editorEngine.webview.deselectAll();
108105
// editorEngine.webview.select(webview);
109106
},
110-
[getWebview, editorEngine.webview],
107+
[editorEngine.webview],
111108
);
112109

113110
function handleDoubleClick(e: React.MouseEvent<HTMLDivElement>) {
@@ -118,15 +115,15 @@ export const GestureScreen = observer(() => {
118115
}
119116

120117
function handleMouseDown(e: React.MouseEvent<HTMLDivElement>) {
121-
// if (editorEngine.state.editorMode === EditorMode.DESIGN) {
122-
// handleMouseEvent(e, MouseAction.MOUSE_DOWN);
123-
// } else if (
124-
// editorEngine.state.editorMode === EditorMode.INSERT_DIV ||
125-
// editorEngine.state.editorMode === EditorMode.INSERT_TEXT ||
126-
// editorEngine.state.editorMode === EditorMode.INSERT_IMAGE
127-
// ) {
128-
// editorEngine.insert.start(e);
129-
// }
118+
if (editorEngine.state.editorMode === EditorMode.DESIGN) {
119+
handleMouseEvent(e, MouseAction.MOUSE_DOWN);
120+
} else if (
121+
editorEngine.state.editorMode === EditorMode.INSERT_DIV ||
122+
editorEngine.state.editorMode === EditorMode.INSERT_TEXT ||
123+
editorEngine.state.editorMode === EditorMode.INSERT_IMAGE
124+
) {
125+
// editorEngine.insert.start(e);
126+
}
130127
}
131128

132129
async function handleMouseUp(e: React.MouseEvent<HTMLDivElement>) {

‎apps/web/client/src/app/project/[id]/_components/canvas/frame/index.tsx

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
import { FrameType, type Frame, type WebFrame } from "@onlook/models";
22
import { observer } from "mobx-react-lite";
3+
import { useRef } from "react";
34
import { GestureScreen } from './gesture';
45
import { ResizeHandles } from './resize-handles';
56
import { TopBar } from "./top-bar";
6-
import { WebFrameComponent } from "./web-frame";
7+
import { WebFrameComponent, type WebFrameView } from "./web-frame";
78

89
export const FrameView = observer(
910
({
1011
frame,
1112
}: {
1213
frame: Frame;
1314
}) => {
15+
const webFrameRef = useRef<WebFrameView>(null);
1416
return (
1517
<div
1618
className="flex flex-col fixed"
@@ -20,8 +22,8 @@ export const FrameView = observer(
2022
</TopBar>
2123
<div className="relative">
2224
<ResizeHandles frame={frame} />
23-
{frame.type === FrameType.WEB && <WebFrameComponent frame={frame as WebFrame} />}
24-
<GestureScreen />
25+
{frame.type === FrameType.WEB && <WebFrameComponent frame={frame as WebFrame} ref={webFrameRef} />}
26+
<GestureScreen frame={frame as WebFrame} webFrameRef={webFrameRef} />
2527
{/* {domFailed && shouldShowDomFailed && renderNotRunning()} */}
2628
</div>
2729

‎apps/web/client/src/app/project/[id]/_components/canvas/frame/web-frame.tsx

Lines changed: 131 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,137 @@
1-
import type { WebFrame } from "@onlook/models";
1+
import type { DomElement, WebFrame } from "@onlook/models";
22
import { cn } from "@onlook/ui-v4/utils";
33
import { observer } from "mobx-react-lite";
4+
import { WindowMessenger, connect } from 'penpal';
5+
import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef, useState, type IframeHTMLAttributes } from 'react';
46

5-
export const WebFrameComponent = observer(({
6-
frame,
7-
}: { frame: WebFrame }) => {
7+
export type WebFrameView = HTMLIFrameElement &
8+
Pick<
9+
Electron.WebviewTag,
10+
| 'setZoomLevel'
11+
| 'loadURL'
12+
| 'openDevTools'
13+
| 'canGoForward'
14+
| 'canGoBack'
15+
| 'goForward'
16+
| 'goBack'
17+
| 'reload'
18+
| 'isLoading'
19+
| 'capturePage'
20+
> & {
21+
supportsOpenDevTools: () => boolean;
22+
capturePageAsCanvas: () => Promise<HTMLCanvasElement>;
23+
getElementAtLoc: (x: number, y: number, getStyle: boolean) => Promise<DomElement>;
24+
};
25+
26+
interface WebFrameViewProps extends IframeHTMLAttributes<HTMLIFrameElement> {
27+
frame: WebFrame;
28+
}
29+
30+
export const WebFrameComponent = observer(forwardRef<WebFrameView, WebFrameViewProps>(({ frame, ...props }, ref) => {
31+
const [iframeRemote, setIframeRemote] = useState<any>(null);
32+
33+
const iframeRef = useRef<HTMLIFrameElement>(null);
34+
const zoomLevel = useRef(1);
35+
36+
const handleIframeLoad = useCallback(() => {
37+
const iframe = iframeRef.current;
38+
if (!iframe) {
39+
console.error('No iframe found');
40+
return;
41+
}
42+
43+
const initializePenpalConnection = async () => {
44+
try {
45+
console.log('Initializing penpal connection for frame', frame.id);
46+
if (!iframe?.contentWindow) {
47+
throw new Error('No content window found');
48+
}
49+
const messenger = new WindowMessenger({
50+
remoteWindow: iframe.contentWindow,
51+
// TODO: Use a proper origin
52+
allowedOrigins: ['*'],
53+
});
54+
const connection = connect({
55+
messenger,
56+
// Methods we are exposing to the iframe window.
57+
methods: {}
58+
});
59+
const remote = await connection.promise as any;
60+
setIframeRemote(remote);
61+
console.log('Penpal connection initialized for frame', frame.id);
62+
} catch (error) {
63+
console.error('Initialize penpal connection failed:', error);
64+
}
65+
};
66+
67+
if (iframe.contentDocument?.readyState === 'complete') {
68+
initializePenpalConnection();
69+
} else {
70+
iframe.addEventListener('load', initializePenpalConnection, { once: true });
71+
}
72+
}, [iframeRemote]);
73+
74+
useEffect(() => {
75+
handleIframeLoad();
76+
}, [handleIframeLoad]);
77+
78+
useImperativeHandle(ref, () => {
79+
const iframe = iframeRef.current!;
80+
81+
Object.assign(iframe, {
82+
supportsOpenDevTools: () => {
83+
const contentWindow = iframe.contentWindow;
84+
return !!contentWindow && 'openDevTools' in contentWindow;
85+
},
86+
openDevTools: () => {
87+
const contentWindow = iframe.contentWindow;
88+
if (!contentWindow || !('openDevTools' in contentWindow)) {
89+
throw new Error('openDevTools() is not supported in this browser');
90+
}
91+
(contentWindow as any as Electron.WebContents).openDevTools();
92+
},
93+
setZoomLevel: (level: number) => {
94+
zoomLevel.current = level;
95+
iframe.style.transform = `scale(${level})`;
96+
iframe.style.transformOrigin = 'top left';
97+
},
98+
loadURL: async (url: string) => {
99+
iframe.src = url;
100+
},
101+
canGoForward: () => {
102+
return (iframe.contentWindow?.history?.length ?? 0) > 0;
103+
},
104+
canGoBack: () => {
105+
return (iframe.contentWindow?.history?.length ?? 0) > 0;
106+
},
107+
goForward: () => {
108+
iframe.contentWindow?.history.forward();
109+
},
110+
goBack: () => {
111+
iframe.contentWindow?.history.back();
112+
},
113+
reload: () => {
114+
iframe.contentWindow?.location.reload();
115+
},
116+
isLoading: (): boolean => {
117+
const contentDocument = iframe.contentDocument;
118+
if (!contentDocument) {
119+
throw new Error(
120+
'Could not call isLoading(): iframe.contentDocument is null/undefined',
121+
);
122+
}
123+
return contentDocument.readyState !== 'complete';
124+
},
125+
getElementAtLoc: async (x: number, y: number, getStyle: boolean) => {
126+
return await iframeRemote?.getElementAtLoc(x, y, getStyle);
127+
},
128+
});
129+
130+
return iframe as WebFrameView;
131+
}, [iframeRemote]);
8132
return (
9133
<iframe
134+
ref={iframeRef}
10135
id={frame.id}
11136
className={cn(
12137
'backdrop-blur-sm transition outline outline-4',
@@ -20,6 +145,7 @@ export const WebFrameComponent = observer(({
20145
width: frame.dimension.width,
21146
height: frame.dimension.height,
22147
}}
148+
{...props}
23149
/>
24150
);
25-
});
151+
}));

‎apps/web/client/src/components/store/editor/engine/canvas/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ export class CanvasManager {
2121

2222
const webFrame: WebFrame = {
2323
id: '1',
24-
url: 'https://react.dev/',
24+
url: 'http://localhost:8084',
2525
position: { x: 0, y: 0 },
2626
dimension: { width: 1000, height: 1000 },
2727
type: FrameType.WEB,

‎apps/web/client/src/components/store/editor/engine/overlay/utils.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import type { WebFrameView } from '@/app/project/[id]/_components/canvas/frame/web-frame';
12
import { EditorAttributes } from '@onlook/models/constants';
23
import type { ElementPosition } from '@onlook/models/element';
34
import type { WebviewTag } from 'electron/renderer';
@@ -85,7 +86,7 @@ export function adaptValueToCanvas(value: number, inverse = false): number {
8586
*/
8687
export function getRelativeMousePositionToWebview(
8788
e: React.MouseEvent<HTMLDivElement>,
88-
webview: WebviewTag,
89+
webview: WebFrameView,
8990
inverse: boolean = false,
9091
): ElementPosition {
9192
const rect = webview.getBoundingClientRect();

‎apps/web/docker-compose.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ services:
4545
context: ./template
4646
dockerfile: Dockerfile
4747
ports:
48-
- "8084:3000"
48+
- "8084:8084"
4949
volumes:
5050
- ./packages/template:/app
5151
- /app/node_modules

0 commit comments

Comments
 (0)