import React, { useState } from "react";
import { useDataProvider, useNotify, Title } from "react-admin";
import { parse as parseCsv, unparse as unparseCsv } from "papaparse";
import {
Button,
Card,
CardActions,
CardContent,
CardHeader,
Checkbox,
Container,
FormControlLabel,
NativeSelect,
} from "@mui/material";
import { useTranslate } from "ra-core";
import { generateRandomUser } from "./users";
const LOGGING = true;
const expectedFields = ["id", "displayname"].sort();
const optionalFields = [
"user_type",
"guest",
"admin",
"deactivated",
"avatar_url",
"password",
].sort();
function TranslatableOption({ value, text }) {
const translate = useTranslate();
return ;
}
const FilePicker = props => {
const [values, setValues] = useState(null);
const [error, setError] = useState(null);
const [stats, setStats] = useState(null);
const [dryRun, setDryRun] = useState(true);
const [progress, setProgress] = useState(null);
const [importResults, setImportResults] = useState(null);
const [skippedRecords, setSkippedRecords] = useState(null);
const [conflictMode, setConflictMode] = useState("stop");
const [passwordMode, setPasswordMode] = useState(true);
const [useridMode, setUseridMode] = useState("ignore");
const translate = useTranslate();
const notify = useNotify();
const dataProvider = useDataProvider();
const onFileChange = async e => {
if (progress !== null) return;
setValues(null);
setError(null);
setStats(null);
setImportResults(null);
const file = e.target.files ? e.target.files[0] : null;
/* Let's refuse some unreasonably big files instead of freezing
* up the browser */
if (file.size > 100000000) {
const message = translate("import_users.errors.unreasonably_big", {
size: (file.size / (1024 * 1024)).toFixed(2),
});
notify(message);
setError(message);
return;
}
try {
parseCsv(file, {
header: true,
skipEmptyLines: true /* especially for a final EOL in the csv file */,
complete: result => {
if (result.error) {
setError(result.error);
}
/* Papaparse is very lenient, we may be able to salvage
* the data in the file. */
verifyCsv(result, { setValues, setStats, setError });
},
});
} catch {
setError(true);
return null;
}
};
const verifyCsv = (
{ data, meta, errors },
{ setValues, setStats, setError }
) => {
/* First, verify the presence of required fields */
let eF = Array.from(expectedFields);
let oF = Array.from(optionalFields);
meta.fields.forEach(name => {
if (eF.includes(name)) {
eF = eF.filter(v => v !== name);
}
if (oF.includes(name)) {
oF = oF.filter(v => v !== name);
}
});
if (eF.length !== 0) {
setError(
translate("import_users.error.required_field", { field: eF[0] })
);
return false;
}
// XXX after deciding on how "name" and friends should be handled below,
// this place will want changes, too.
/* Collect some stats to prevent sneaky csv files from adding admin
users or something.
*/
let stats = {
user_types: { default: 0 },
is_guest: 0,
admin: 0,
deactivated: 0,
password: 0,
avatar_url: 0,
id: 0,
total: data.length,
};
data.forEach((line, idx) => {
if (line.user_type === undefined || line.user_type === "") {
stats.user_types.default++;
} else {
stats.user_types[line.user_type] += 1;
}
/* XXX correct the csv export that react-admin offers for the users
* resource so it gives sensible field names and doesn't duplicate
* id as "name"?
*/
if (meta.fields.includes("name")) {
delete line.name;
}
if (meta.fields.includes("user_type")) {
delete line.user_type;
}
if (meta.fields.includes("is_admin")) {
line.admin = line.is_admin;
delete line.is_admin;
}
["is_guest", "admin", "deactivated"].forEach(f => {
if (line[f] === "true") {
stats[f]++;
line[f] = true; // we need true booleans instead of strings
} else {
if (line[f] !== "false" && line[f] !== "") {
errors.push(
translate("import_users.error.invalid_value", {
field: f,
row: idx,
})
);
}
line[f] = false; // default values to false
}
});
if (line.password !== undefined && line.password !== "") {
stats.password++;
}
if (line.avatar_url !== undefined && line.avatar_url !== "") {
stats.avatar_url++;
}
if (line.id !== undefined && line.id !== "") {
stats.id++;
}
});
if (errors.length > 0) {
setError(errors);
}
setStats(stats);
setValues(data);
return true;
};
const runImport = async e => {
if (progress !== null) {
notify("import_users.errors.already_in_progress");
return;
}
const results = await doImport(
dataProvider,
values,
conflictMode,
passwordMode,
useridMode,
dryRun,
setProgress,
setError
);
setImportResults(results);
// offer CSV download of skipped or errored records
// (so that the user doesn't have to filter out successful
// records manually when fixing stuff in the CSV)
setSkippedRecords(unparseCsv(results.skippedRecords));
if (LOGGING) console.log("Skipped records:");
if (LOGGING) console.log(skippedRecords);
};
// XXX every single one of the requests will restart the activity indicator
// which doesn't look very good.
const doImport = async (
dataProvider,
data,
conflictMode,
passwordMode,
useridMode,
dryRun,
setProgress,
setError
) => {
let skippedRecords = [];
let erroredRecords = [];
let succeededRecords = [];
let changeStats = {
toAdmin: 0,
toGuest: 0,
toRegular: 0,
replacedPassword: 0,
};
let entriesDone = 0;
let entriesCount = data.length;
try {
setProgress({ done: entriesDone, limit: entriesCount });
for (const entry of data) {
let userRecord = {};
let overwriteData = {};
// No need to do a bunch of cryptographic random number getting if
// we are using neither a generated password nor a generated user id.
if (
useridMode === "ignore" ||
entry.id === undefined ||
entry.password === undefined ||
passwordMode === false
) {
overwriteData = generateRandomUser();
// Ignoring IDs or the entry lacking an ID means we keep the
// ID field in the overwrite data.
if (!(useridMode === "ignore" || entry.id === undefined)) {
delete overwriteData.id;
}
// Not using passwords from the csv or this entry lacking a password
// means we keep the password field in the overwrite data.
if (
!(
passwordMode === false ||
entry.password === undefined ||
entry.password === ""
)
) {
delete overwriteData.password;
}
}
/* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */
Object.assign(userRecord, entry);
Object.assign(userRecord, overwriteData);
/* For these modes we will consider the ID that's in the record.
* If the mode is "stop", we will not continue adding more records, and
* we will offer information on what was already added and what was
* skipped.
*
* If the mode is "skip", we record the record for later, but don't
* send it to the server.
*
* If the mode is "update", we change fields that are reasonable to
* update.
* - If the "password mode" is "true" (i.e. "use passwords from csv"):
* - if the record has a password
* - send the password along with the record
* - if the record has no password
* - generate a new password
* - If the "password mode" is "false"
* - never generate a new password to update existing users with
*/
/* We just act as if there are no IDs in the CSV, so every user will be
* created anew.
* We do a simple retry loop so that an accidental hit on an existing ID
* doesn't trip us up.
*/
if (LOGGING)
console.log(
"will check for existence of record " + JSON.stringify(userRecord)
);
let retries = 0;
const submitRecord = recordData => {
return dataProvider.getOne("users", { id: recordData.id }).then(
async alreadyExists => {
if (LOGGING) console.log("already existed");
if (useridMode === "update" || conflictMode === "skip") {
skippedRecords.push(recordData);
} else if (conflictMode === "stop") {
throw new Error(
translate("import_users.error.id_exits", {
id: recordData.id,
})
);
} else {
const overwriteData = generateRandomUser();
const newRecordData = Object.assign({}, recordData, {
id: overwriteData.id,
});
retries++;
if (retries > 512) {
console.warn("retry loop got stuck? pathological situation?");
skippedRecords.push(recordData);
} else {
await submitRecord(newRecordData);
}
}
},
async okToSubmit => {
if (LOGGING)
console.log(
"OK to create record " +
recordData.id +
" (" +
recordData.displayname +
")."
);
if (!dryRun) {
await dataProvider.create("users", { data: recordData });
}
succeededRecords.push(recordData);
}
);
};
await submitRecord(userRecord);
entriesDone++;
setProgress({ done: entriesDone, limit: data.length });
}
setProgress(null);
} catch (e) {
setError(
translate("import_users.error.at_entry", {
entry: entriesDone + 1,
message: e.message,
})
);
setProgress(null);
}
return {
skippedRecords,
erroredRecords,
succeededRecords,
totalRecordCount: entriesCount,
changeStats,
wasDryRun: dryRun,
};
};
const downloadSkippedRecords = () => {
const element = document.createElement("a");
console.log(skippedRecords);
const file = new Blob([skippedRecords], {
type: "text/comma-separated-values",
});
element.href = URL.createObjectURL(file);
element.download = "skippedRecords.csv";
document.body.appendChild(element); // Required for this to work in FireFox
element.click();
};
const onConflictModeChanged = async e => {
if (progress !== null) {
return;
}
const value = e.target.value;
setConflictMode(value);
};
const onPasswordModeChange = e => {
if (progress !== null) {
return;
}
setPasswordMode(e.target.checked);
};
const onUseridModeChanged = async e => {
if (progress !== null) {
return;
}
const value = e.target.value;
setUseridMode(value);
};
const onDryRunModeChanged = ev => {
if (progress !== null) {
return;
}
setDryRun(ev.target.checked);
};
// render individual small components
const statsCards = stats &&
!importResults && [