Skip to content

Detect file encoding from byte order mark #235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dronov-dmitry wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Detect file encoding from byte order mark #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0aca26c
Detect text file encoding from byte order mark
dronov-dmitry Aug 17, 2026
84ad11c
Persist encoding/BOM in recovery metadata and reconstruct in restoreD…
dronov-dmitry Sep 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions app/src/androidTest/java/com/maxistar/textpad/test/EditorRecoveryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,50 @@ public void successfulSaveAsRemovesUntitledRecoveryAndUsesNamedIdentity() throws
}
}

@Test
public void utf16leWithBomRoundTripsThroughRecoveryRestoreAndSave() throws Exception {
Assume.assumeTrue(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q);
String content = "Hello UTF-16LE";
byte[] utf16leBytes = content.getBytes(java.nio.charset.Charset.forName("UTF-16LE"));
byte[] withBom = new byte[2 + utf16leBytes.length];
withBom[0] = (byte) 0xFF;
withBom[1] = (byte) 0xFE;
java.lang.System.arraycopy(utf16leBytes, 0, withBom, 2, utf16leBytes.length);

Uri documentUri = createTestDocumentRaw(withBom);
String key = RecoveryKeys.forDocumentUri(documentUri.toString());
new RecoveryRepository(context).write(
metadataWithEncoding(key, documentUri.toString(), "UTF-16LE", true),
content
);

Intent intent = new Intent(context, EditorActivity.class)
.setAction(Intent.ACTION_VIEW)
.setData(documentUri);

try (ActivityScenario<EditorActivity> scenario = ActivityScenario.launch(intent)) {
onView(withText(R.string.Restore)).perform(click());
onView(withId(R.id.editText1)).check(matches(withText(content)));
scenario.onActivity(activity -> {
EditText editor = activity.findViewById(R.id.editText1);
editor.setSelection(content.length());
editor.getText().append(" modified");
});
android.os.SystemClock.sleep(800);
scenario.recreate();
onView(withText(R.string.Restore)).perform(click());
scenario.onActivity(activity -> invokeNoArgument(activity, "saveNamedFile"));
byte[] savedBytes = readDocumentRawBytes(documentUri);
assertEquals(0xFF, savedBytes[0] & 0xFF);
assertEquals(0xFE, savedBytes[1] & 0xFF);
String savedText = new String(savedBytes, 2, savedBytes.length - 2,
java.nio.charset.Charset.forName("UTF-16LE"));
assertEquals("Hello UTF-16LE modified", savedText);
} finally {
context.getContentResolver().delete(documentUri, null, null);
}
}

private void assertLargeDocumentStateIsBinderSafe(boolean simpleScrolling) {
setSimpleScrolling(simpleScrolling);
String content = generatedDocument(1_050_000);
Expand Down Expand Up @@ -261,6 +305,15 @@ private RecoveryMetadata metadata(String key, String documentUri) {
);
}

private RecoveryMetadata metadataWithEncoding(String key, String documentUri,
String encoding, boolean hasBom) {
return new RecoveryMetadata(
key, documentUri, documentUri == null ? "newfile.txt" : "notes.txt",
documentUri == null, encoding, hasBom,
null, null, null, 0, 0, 0, 0, 1
);
}

private Uri createTestDocument(String content) throws Exception {
ContentValues values = new ContentValues();
values.put(MediaStore.MediaColumns.DISPLAY_NAME, "textpad-recovery-" + java.lang.System.nanoTime() + ".txt");
Expand Down Expand Up @@ -296,6 +349,39 @@ private String readDocument(Uri uri) {
}
}

private Uri createTestDocumentRaw(byte[] rawContent) throws Exception {
ContentValues values = new ContentValues();
values.put(MediaStore.MediaColumns.DISPLAY_NAME, "textpad-recovery-" + java.lang.System.nanoTime() + ".txt");
values.put(MediaStore.MediaColumns.MIME_TYPE, "text/plain");
values.put(MediaStore.MediaColumns.RELATIVE_PATH, "Download/TextPadTests");
Uri uri = context.getContentResolver().insert(MediaStore.Downloads.EXTERNAL_CONTENT_URI, values);
if (uri == null) {
throw new IllegalStateException("Unable to create test document");
}
try (java.io.OutputStream output = context.getContentResolver().openOutputStream(uri, "wt")) {
if (output == null) {
throw new IllegalStateException("Unable to write test document");
}
output.write(rawContent);
}
return uri;
}

private byte[] readDocumentRawBytes(Uri uri) throws Exception {
try (InputStream input = context.getContentResolver().openInputStream(uri);
ByteArrayOutputStream output = new ByteArrayOutputStream()) {
if (input == null) {
throw new IllegalStateException("Unable to read test document");
}
byte[] buffer = new byte[1024];
int count;
while ((count = input.read(buffer)) != -1) {
output.write(buffer, 0, count);
}
return output.toByteArray();
}
}

private static void invokeNoArgument(EditorActivity activity, String methodName) {
try {
Method method = EditorActivity.class.getDeclaredMethod(methodName);
Expand Down
47 changes: 34 additions & 13 deletions app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import com.maxistar.textpad.recovery.RecoveryWriter;
import com.maxistar.textpad.utils.EditTextUndoRedo;
import com.maxistar.textpad.utils.DocumentSaveValidator;
import com.maxistar.textpad.utils.FileEncoding;
import com.maxistar.textpad.utils.FileNameHelper;
import com.maxistar.textpad.utils.System;
import com.maxistar.textpad.utils.TextConverter;
Expand Down Expand Up @@ -130,6 +131,8 @@ public class EditorActivity extends AppCompatActivity {
private ScrollView scrollView;
private LinearLayout linearLayout;

private FileEncoding documentEncoding;

String urlFilename = TPStrings.EMPTY;

Uri lastTriedSystemUri = null;
Expand Down Expand Up @@ -261,6 +264,13 @@ private boolean simpleScrolling() {
return settingsService.isUseSimpleScrolling();
}

private String resolveFileEncodingName() {
if (documentEncoding != null) {
return documentEncoding.getCharsetName();
}
return settingsService.getFileEncoding();
}

@RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM)
private void applyEdgeToEdgeInsets() {
View editorRoot = findViewById(R.id.editor_root);
Expand Down Expand Up @@ -534,6 +544,9 @@ private void restoreDraft(RecoveryDraft draft) {
originalSize = draft.metadata.originalSize;
originalLastModified = draft.metadata.originalLastModified;
originalContentSha256 = draft.metadata.originalContentSha256;
if (draft.metadata.encoding != null && !draft.metadata.encoding.isEmpty()) {
documentEncoding = FileEncoding.fromCharset(draft.metadata.encoding, draft.metadata.hasBom);
}
setEditorText(draft.text, true);
updateTitle();
if (!draft.metadata.untitled) {
Expand Down Expand Up @@ -594,8 +607,8 @@ private RecoveryWriter.Snapshot createRecoverySnapshot() {
identity,
currentDisplayName(),
identity == null,
settingsService.getFileEncoding(),
false,
resolveFileEncodingName(),
documentEncoding != null && documentEncoding.hasBom(),
originalSize,
originalLastModified,
originalContentSha256,
Expand Down Expand Up @@ -1159,6 +1172,7 @@ public void clearFile() {
originalSize = null;
originalLastModified = null;
originalContentSha256 = null;
documentEncoding = null;
selectionStart = 0;
selectionEnd = 0;
setEditorText(TPStrings.EMPTY, false);
Expand Down Expand Up @@ -1340,7 +1354,7 @@ protected void saveFile(Uri uri) throws IOException {

s = applyEndings(s);

outputStream.write(s.getBytes(settingsService.getFileEncoding()));
outputStream.write(FileEncoding.encode(s, documentEncoding, settingsService.getFileEncoding()));
} finally {
outputStream.close();
}
Expand All @@ -1356,7 +1370,7 @@ private void guardedSaveNamedFile(boolean autosave) {
SaveRequest request = new SaveRequest(
editorGeneration,
recoveryKey,
persistedText.getBytes(settingsService.getFileEncoding())
FileEncoding.encode(persistedText, documentEncoding, settingsService.getFileEncoding())
);
boolean creatingDocument = nextSaveCreatesDocument || originalContentSha256 == null;
nextSaveCreatesDocument = false;
Expand Down Expand Up @@ -1551,8 +1565,11 @@ private void validateOpenDocumentOnForeground() {
return;
}

byte[] intendedBytes = applyEndings(mText.getText().toString())
.getBytes(settingsService.getFileEncoding());
byte[] intendedBytes = FileEncoding.encode(
applyEndings(mText.getText().toString()),
documentEncoding,
settingsService.getFileEncoding()
);
SaveRequest request = new SaveRequest(editorGeneration, recoveryKey, intendedBytes);
DocumentSaveValidator.Outcome outcome = DocumentSaveValidator.classify(
currentBytes,
Expand All @@ -1572,7 +1589,8 @@ private void validateOpenDocumentOnForeground() {
}

private void applyExternalDocument(byte[] externalBytes) throws Exception {
String externalText = new String(externalBytes, settingsService.getFileEncoding());
documentEncoding = FileEncoding.detect(externalBytes);
String externalText = FileEncoding.decode(externalBytes, documentEncoding, settingsService.getFileEncoding());
externalText = toUnixEndings(externalText);
setEditorText(externalText, false);
initEditor();
Expand All @@ -1588,8 +1606,11 @@ private void validateRestoredDraft() {
return;
}
try {
byte[] intendedBytes = applyEndings(mText.getText().toString())
.getBytes(settingsService.getFileEncoding());
byte[] intendedBytes = FileEncoding.encode(
applyEndings(mText.getText().toString()),
documentEncoding,
settingsService.getFileEncoding()
);
SaveRequest request = new SaveRequest(editorGeneration, recoveryKey, intendedBytes);
byte[] currentBytes = readNamedDocumentBytes();
DocumentSaveValidator.Outcome outcome = DocumentSaveValidator.classify(
Expand Down Expand Up @@ -1664,9 +1685,8 @@ private void openNamedFileLegacyDirect(String filename) {
dis.close();
fis.close();

String ttt = new String(b, 0, length,
settingsService.getFileEncoding());

documentEncoding = FileEncoding.detect(b);
String ttt = FileEncoding.decode(b, documentEncoding, settingsService.getFileEncoding());
ttt = toUnixEndings(ttt);

setEditorText(ttt, false);
Expand Down Expand Up @@ -1717,7 +1737,8 @@ private void openNamedFileDirect(final Uri uri) {
}
byte[] b = bytes.toByteArray();

String ttt = new String(b, settingsService.getFileEncoding());
documentEncoding = FileEncoding.detect(b);
String ttt = FileEncoding.decode(b, documentEncoding, settingsService.getFileEncoding());
ttt = toUnixEndings(ttt);

inputStream.close();
Expand Down
12 changes: 10 additions & 2 deletions app/src/main/java/com/maxistar/textpad/recovery/RecoveryMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ public JSONObject toJson() throws JSONException {
value.put("documentUri", documentUri == null ? JSONObject.NULL : documentUri);
value.put("displayName", displayName);
value.put("isUntitled", untitled);
value.put("encoding", encoding);
value.put("encoding", encoding == null ? "" : encoding);
value.put("hasBom", hasBom);
value.put("originalSize", originalSize == null ? JSONObject.NULL : originalSize);
value.put("originalLastModified", originalLastModified == null ? JSONObject.NULL : originalLastModified);
Expand All @@ -169,7 +169,7 @@ public static RecoveryMetadata fromJson(JSONObject value) throws JSONException {
nullableString(value, "documentUri"),
value.optString("displayName", ""),
value.getBoolean("isUntitled"),
value.optString("encoding", "UTF-8"),
nullableStringWithFallback(value, "encoding", "UTF-8"),
value.optBoolean("hasBom", false),
nullableLong(value, "originalSize"),
nullableLong(value, "originalLastModified"),
Expand All @@ -187,6 +187,14 @@ private static String nullableString(JSONObject value, String name) throws JSONE
return value.isNull(name) ? null : value.getString(name);
}

private static String nullableStringWithFallback(JSONObject value, String name, String fallback) throws JSONException {
if (value.isNull(name) || !value.has(name)) {
return fallback;
}
String result = value.getString(name);
return result == null || result.isEmpty() ? fallback : result;
}

private static Long nullableLong(JSONObject value, String name) throws JSONException {
return value.isNull(name) ? null : value.getLong(name);
}
Expand Down
Loading