8000 API importer for Notion by hmacr · Pull Request #8710 · outline/outline · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content
8000

API importer for Notion #8710

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 54 commits into from
Mar 23, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
b7e6d36
config driven imports page
hmacr Mar 5, 2025
0914dbf
existing notion importer as plugin
hmacr Mar 5, 2025
c471046
oauth redirect
hmacr Mar 5, 2025
d1eb948
exchange code for access_token and persist
hmacr Mar 5, 2025
a36b7a6
show oauth errors
hmacr Mar 6, 2025
c294f41
list root pages in a dialog
hmacr Mar 6, 2025
90a9b14
basic table UI
hmacr Mar 6, 2025
0b35e06
create import
hmacr Mar 6, 2025
0ba22c2
remove local oauth redirect
hmacr Mar 6, 2025
7974d58
ui
hmacr Mar 6, 2025
c73f6a4
API importer backend (#8664)
hmacr Mar 16, 2025
e8a2229
Add link_to_page support
tommoor Mar 17, 2025
eac7c37
lint
hmacr Mar 17, 2025
2eda9da
i18n
hmacr Mar 17, 2025
0011bfd
collection content overwritten in collections.list API
hmacr Mar 17, 2025
9ddaf41
load latest policy when import completes
hmacr Mar 17, 2025
50639e7
doc source metadata
hmacr Mar 17, 2025
33f278b
Merge branch 'main' into notion-importer
hmacr Mar 17, 2025
8f06f35
cleanup name
hmacr Mar 17, 2025
d71ad7a
tsc
hmacr Mar 17, 2025
b95c3a7
CleanupOldImportTasks task
hmacr Mar 17, 2025
0e21aad
ErrorTimedOutImports task
hmacr Mar 17, 2025
03d4132
restrict to 1 active import per team
hmacr Mar 17, 2025
1dc9a7a
remove constants file
hmacr Mar 18, 2025
3f7280c
move oauth to NotionClient
hmacr Mar 18, 2025
98718dd
rename policy
hmacr Mar 18, 2025
67e4d43
better queries
hmacr Mar 18, 2025
d69ea5d
fileOp item key
hmacr Mar 18, 2025
e9de4ac
fetch import with lock
hmacr Mar 18, 2025
c1bb509
buffer issue
hmacr Mar 18, 2025
68a45fd
better docStructure update
hmacr Mar 18, 2025
432714d
teamId
hmacr Mar 18, 2025
3f98207
fix table indexes
hmacr Mar 18, 2025
d793b66
jsdoc
hmacr Mar 18, 2025
0ec97c9
suppress websocket events for imports
hmacr Mar 18, 2025
659a383
fix NotionConverter test
hmacr Mar 18, 2025
11544b3
move NotionConverter inside plugin
hmacr Mar 18, 2025
a1b3c9d
rename pageCount to documentCount
hmacr Mar 18, 2025
015a9bb
re-throw upload attachments error for retry
hmacr Mar 18, 2025
bb3f041
handle child_page, child_database in NotionConverter
hmacr Mar 18, 2025
b730c5c
cancel import from UI
hmacr Mar 18, 2025
6d92053
modify cleanup tasks to account for canceled state
hmacr Mar 19, 2025
2ed98e7
remove obsolete snapshot
hmacr Mar 19, 2025
0693b6a
tests
hmacr Mar 19, 2025
8327d4b
handle bot owner NA case
hmacr Mar 20, 2025
04db4f5
timeout error
hmacr Mar 21, 2025
25edb28
use createdAt, updatedAt from Notion
hmacr Mar 21, 2025
d9f8cda
load root pages in processor
hmacr Mar 22, 2025
eb345bc
handle empty doc
hmacr Mar 22, 2025
e8f9e9e
tests
hmacr Mar 22, 2025
ba89f9f
Merge branch 'main' into notion-importer
tommoor Mar 23, 2025
1027e93
cleanup
tommoor Mar 23, 2025
cec1aef
Improve usePolicy hook
tommoor Mar 23, 2025
f99507d
fix: Mark NOTION_ env as optional
tommoor Mar 23, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .env.sample
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,10 @@ DISCORD_SERVER_ID=
# DISCORD_SERVER_ID and DISCORD_SERVER_ROLES must be set together.
DISCORD_SERVER_ROLES=

# –––––––––––––– IMPORTS ––––––––––––––
NOTION_CLIENT_ID=
NOTION_CLIENT_SECRET=

# –––––––––––––––– OPTIONAL ––––––––––––––––

# Base64 encoded private key and certificate for HTTPS termination. This is only
Expand Down
5 changes: 4 additions & 1 deletion app/components/Dialogs.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@ function Dialogs() {
key={id}
isOpen={modal.isOpen}
fullscreen={modal.fullscreen ?? false}
=> dialogs.closeModal(id)}
=> {
modal.onClose?.();
dialogs.closeModal(id);
}}
title={modal.title}
style={modal.style}
>
Expand Down
25 changes: 24 additions & 1 deletion app/components/WebsocketProvider.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ import * as React from "react";
import { withTranslation, WithTranslation } from "react-i18next";
import { io, Socket } from "socket.io-client";
import { toast } from "sonner";
import { FileOperationState, FileOperationType } from "@shared/types";
import {
FileOperationState,
FileOperationType,
ImportState,
} from "@shared/types";
import RootStore from "~/stores/RootStore";
import Collection from "~/models/Collection";
import Comment from "~/models/Comment";
Expand All @@ -15,6 +19,7 @@ import FileOperation from "~/models/FileOperation";
import Group from "~/models/Group";
import GroupMembership from "~/models/GroupMembership";
import GroupUser from "~/models/GroupUser";
import Import from "~/models/Import";
import Membership from "~/models/Membership";
import Notification from "~/models/Notification";
import Pin from "~/models/Pin";
Expand Down Expand Up @@ -100,6 +105,7 @@ class WebsocketProvider extends React.Component<Props> {
subscriptions,
fileOperations,
notifications,
imports,
} = this.props;

const currentUserId = auth?.user?.id;
Expand Down Expand Up @@ -620,6 +626,23 @@ class WebsocketProvider extends React.Component<Props> {
}
);

this.socket.on("imports.create", (event: PartialExcept<Import, "id">) => {
imports.add(event);
});

this.socket.on("imports.update", (event: PartialExcept<Import, "id">) => {
imports.add(event);

if (
event.state === ImportState.Completed &&
event.createdBy?.id === auth.user?.id
) {
toast.success(event.name, {
description: this.props.t("Your import completed"),
});
}
});

this.socket.on(
"subscriptions.create",
(event: PartialExcept<Subscription, "id">) => {
Expand Down
5 changes: 3 additions & 2 deletions app/hooks/usePolicy.ts
CEB7
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export default function usePolicy(entity?: string | Model | null) {
? entity
: entity.id
: "";
const policy = policies.get(entityId);

React.useEffect(() => {
if (
Expand All @@ -28,11 +29,11 @@ export default function usePolicy(entity?: string | Model | null) {
) {
// The policy for this model is missing and we have an authenticated session, attempt to
// reload relationships for this model.
if (!policies.get(entity.id) && user) {
if (!policy && user) {
void entity.loadRelations();
}
}
}, [policies, user, entity]);
}, [policies, policy, user, entity]);

return policies.abilities(entityId);
}
62 changes: 62 additions & 0 deletions app/menus/ImportMenu.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import { observer } from "mobx-react";
import { CrossIcon, TrashIcon } from "outline-icons";
import * as React from "react";
import { useTranslation } from "react-i18next";
import { useMenuState } from "reakit/Menu";
import Import from "~/models/Import";
import ContextMenu from "~/components/ContextMenu";
import OverflowMenuButton from "~/components/ContextMenu/OverflowMenuButton";
import Template from "~/components/ContextMenu/Template";
import usePolicy from "~/hooks/usePolicy";
import { MenuItem } from "~/types";

type Props = {
/** Import to which actions will be applied. */
importModel: Import;
/** Callback to handle import cancellation. */
onCancel: () => Promise<void>;
/** Callback to handle import deletion. */
onDelete: () => Promise<void>;
};

export const ImportMenu = observer(
({ importModel, onCancel, onDelete }: Props) => {
const { t } = useTranslation();
const can = usePolicy(importModel);
const menu = useMenuState({
modal: true,
});

const items = React.useMemo(
() =>
[
{
type: "button",
title: t("Cancel"),
visible: can.cancel,
icon: <CrossIcon />,
dangerous: true,
onClick: onCancel,
},
{
type: "button",
title: t("Delete"),
visible: can.delete,
icon: <TrashIcon />,
dangerous: true,
onClick: onDelete,
},
] satisfies MenuItem[],
[t, can.delete, can.cancel, onCancel, onDelete]
);

return (
<>
<OverflowMenuButton aria-label={t("Show menu")} {...menu} />
<ContextMenu {...menu} aria-label={t("Import menu options")}>
<Template {...menu} items={items} />
</ContextMenu>
</>
);
}
);
54 changes: 54 additions & 0 deletions app/models/Import.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import { observable } from "mobx";
import { ImportableIntegrationService, ImportState } from "@shared/types";
import ImportsStore from "~/stores/ImportsStore";
import User from "./User";
import Model from "./base/Model";
import Field from "./decorators/Field";
import { AfterChange } from "./decorators/Lifecycle";
import Relation from "./decorators/Relation";

class Import extends Model {
static modelName = "Import";

store: ImportsStore;

/** The name of the import. */
name: string;

/** The current state of the import. */
@Field
@observable
state: ImportState;

/** The external service from which the import is created. */
service: ImportableIntegrationService;

/** The count of documents created in the import. */
@observable
documentCount: number;

/** The user who created the import. */
@Relation(() => User, {})
createdBy: User;

/** The ID of the user who created the import. */
createdById: string;

/**
* Cancel the import – this will stop the import process and mark it as
* cancelled at the first opportunity.
*/
cancel = async () => this.store.cancel(this);

// hooks

@AfterChange
static removePolicies(model: Import, previousAttributes: Partial<Import>) {
if (previousAttributes.state && previousAttributes.state !== model.state) {
const { policies } = model.store.rootStore;
policies.remove(model.id);
}
}
}

export default Import;
28 changes: 14 additions & 14 deletions app/models/base/Model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,27 +43,27 @@ export default abstract class Model {
this: Model,
options: { withoutPolicies?: boolean } = {}
): Promise<any> {
const relations = getRelationsForModelClass(
this.constructor as typeof Model
);
if (!relations) {
return;
}
// this is to ensure that multiple loads don’t happen in parallel
if (this.loadingRelations) {
return this.loadingRelations;
}

const promises = [];

for (const properties of relations.values()) {
const store = this.store.rootStore.getStoreForModelName(
properties.relationClassResolver().modelName
);
if ("fetch" in store) {
const id = this[properties.idKey];
if (id) {
promises.push(store.fetch(id as string));
const relations = getRelationsForModelClass(
this.constructor as typeof Model
);

if (relations) {
for (const properties of relations.values()) {
const store = this.store.rootStore.getStoreForModelName(
properties.relationClassResolver().modelName
);
if ("fetch" in store) {
const id = this[properties.idKey];
if (id) {
promises.push(store.fetch(id as string));
}
}
}
}
Expand Down
Loading
Loading
0