Skip to content
Permalink
Browse files

Implemented slightly improved parser for TXTs (see #26)

  • Loading branch information...
collinsmith committed Sep 29, 2019
1 parent 28c6bd9 commit d2b48b811960d4a9a11872d9d27ed3d778a10838
@@ -41,7 +41,6 @@
import com.riiablo.codec.Index;
import com.riiablo.codec.Palette;
import com.riiablo.codec.StringTBLs;
import com.riiablo.codec.TXT;
import com.riiablo.console.RenderedConsole;
import com.riiablo.cvar.Cvar;
import com.riiablo.cvar.CvarStateAdapter;
@@ -52,7 +51,6 @@
import com.riiablo.loader.DCCLoader;
import com.riiablo.loader.IndexLoader;
import com.riiablo.loader.PaletteLoader;
import com.riiablo.loader.TXTLoader;
import com.riiablo.map.DS1;
import com.riiablo.map.DS1Loader;
import com.riiablo.map.DT1;
@@ -257,7 +255,6 @@ public void create() {
assets.setLoader(FontTBL.BitmapFont.class, new BitmapFontLoader(mpqs));
assets.setLoader(DT1.class, new DT1Loader(mpqs));
assets.setLoader(DS1.class, new DS1Loader(mpqs));
assets.setLoader(TXT.class, new TXTLoader(mpqs));
assets.setLoader(Map.class, new MapLoader(mpqs));

Riiablo.palettes = palettes = new Palettes(assets);
@@ -12,9 +12,10 @@
import com.badlogic.gdx.utils.ObjectMap;
import com.badlogic.gdx.utils.ObjectSet;
import com.badlogic.gdx.utils.StreamUtils;
import com.riiablo.codec.TXT;
import com.riiablo.util.ClassUtils;

import org.apache.commons.io.IOUtils;

import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataOutput;
@@ -131,8 +132,18 @@

@SuppressWarnings("unchecked")
private static <T extends Excel> T loadTxt(FileHandle handle, Class<T> excelClass, Class<Entry> entryClass, ObjectSet<String> ignore) throws Exception {
TXT txt = TXT.loadFromFile(handle);
TxtParser in = null;
try {
in = TxtParser.loadFromFile(handle);
return loadTxt(in, excelClass, entryClass, ignore);
} catch (Throwable t) {
throw new GdxRuntimeException("Couldn't load excel: " + handle, t);
} finally {
IOUtils.closeQuietly(in);
}
}

private static <T extends Excel> T loadTxt(TxtParser in, Class<T> excelClass, Class<Entry> entryClass, ObjectSet<String> ignore) throws Exception {
final boolean index = ClassUtils.hasAnnotation(entryClass, Index.class);

Field primaryKey = null, firstKey = null;
@@ -182,11 +193,11 @@
columnNames[i] = name;
}

columns.put(field, txt.getColumnId(columnNames));
columns.put(field, in.getColumnId(columnNames));
} else if (startIndex == 0 && endIndex == 0) {
if (DEBUG_COLS) Gdx.app.debug(TAG, fieldName);
TMP[0] = fieldName;
columns.put(field, txt.getColumnId(TMP));
columns.put(field, in.getColumnId(TMP));
} else {
String[] columnNames = new String[endIndex - startIndex];
for (int i = startIndex, j = 0; i < endIndex; i++, j++) {
@@ -195,12 +206,12 @@
columnNames[j] = name;
}

columns.put(field, txt.getColumnId(columnNames));
columns.put(field, in.getColumnId(columnNames));
}
} else {
if (startIndex == 0 && endIndex == 0) {
TMP[0] = format;
columns.put(field, txt.getColumnId(TMP));
columns.put(field, in.getColumnId(TMP));
} else {
String[] columnNames = new String[endIndex - startIndex];
if (values.length == 0) {
@@ -217,7 +228,7 @@
}
}

columns.put(field, txt.getColumnId(columnNames));
columns.put(field, in.getColumnId(columnNames));
}
}
}
@@ -239,68 +250,67 @@

final int primaryKeyCol = index ? -1 : columns.get(primaryKey)[0];
final Class primaryKeyType = index ? null : primaryKey.getType();
final int size = txt.getRows();
for (int i = 0, j = excel.offset(); i < size; i++) {
for (int j = excel.offset(); in.nextLine() != null;) {
Entry entry = entryClass.newInstance();

String rowName = txt.getRowName(i);
String rowName = in.getString(0);
if (ignore.contains(rowName)) {
if (DEBUG_IGNORED) Gdx.app.debug(TAG, "Skipping row " + i + ", ignoring rows named " + rowName);
if (DEBUG_IGNORED) Gdx.app.debug(TAG, "Skipping row " + in.getIndex() + ", ignoring rows named " + rowName);
continue;
}

String name = index ? null : txt.getString(i, primaryKeyCol);
String name = index ? null : in.getString(primaryKeyCol);
for (ObjectMap.Entry<Field, int[]> row : columns.entries()) {
Field field = row.key;
int[] columnIds = row.value;
Class type = field.getType();
assert type.isArray() || columnIds.length == 1 : "field should only correspond to 1 column: " + field.getName() + ", " + columnIds.length + " columns (is it supposed to be an array?)";
if (type == String.class) {
String value = txt.getString(i, columnIds[0]);
String value = in.getString(columnIds[0]);
field.set(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), value));
} else if (type == String[].class) {
String[] value = txt.getString(i, columnIds);
String[] value = in.getString(columnIds);
field.set(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), Arrays.toString(value)));
} else if (type == byte.class) {
byte value = txt.getByte(i, columnIds[0]);
byte value = in.getByte(columnIds[0]);
field.setByte(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), value));
} else if (type == byte[].class) {
byte[] value = txt.getByte(i, columnIds);
byte[] value = in.getByte(columnIds);
field.set(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), Arrays.toString(value)));
} else if (type == short.class) {
short value = txt.getShort(i, columnIds[0]);
short value = in.getShort(columnIds[0]);
field.setShort(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), value));
} else if (type == short[].class) {
short[] value = txt.getShort(i, columnIds);
short[] value = in.getShort(columnIds);
field.set(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), Arrays.toString(value)));
} else if (type == int.class) {
int value = txt.getInt(i, columnIds[0]);
int value = in.getInt(columnIds[0]);
field.setInt(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), value));
} else if (type == int[].class) {
int[] value = txt.getInt(i, columnIds);
int[] value = in.getInt(columnIds);
field.set(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), Arrays.toString(value)));
} else if (type == long.class) {
long value = txt.getLong(i, columnIds[0]);
long value = in.getLong(columnIds[0]);
field.setLong(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), value));
} else if (type == long[].class) {
long[] value = txt.getLong(i, columnIds);
long[] value = in.getLong(columnIds);
field.set(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), Arrays.toString(value)));
} else if (type == boolean.class) {
boolean value = txt.getBoolean(i, columnIds[0]);
boolean value = in.getBoolean(columnIds[0]);
field.setBoolean(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), value));
} else if (type == boolean[].class) {
boolean[] value = txt.getBoolean(i, columnIds);
boolean[] value = in.getBoolean(columnIds);
field.set(entry, value);
if (DEBUG_ENTRIES) Gdx.app.debug(TAG, String.format("Entry[%d](%s).%s=%s", j, name, field.getName(), Arrays.toString(value)));
} else {
@@ -0,0 +1,206 @@
package com.riiablo.codec.excel;

import com.badlogic.gdx.Gdx;
import com.badlogic.gdx.files.FileHandle;
import com.badlogic.gdx.utils.GdxRuntimeException;
import com.badlogic.gdx.utils.ObjectIntMap;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;

import java.io.BufferedReader;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Arrays;

public class TxtParser implements Closeable {
private static final String TAG = "TxtParser";

private static final boolean DEBUG = !true;
private static final boolean DEBUG_COLS = DEBUG && true;
private static final boolean DEBUG_ROWS = DEBUG && true;

private static final boolean FORCE_BOOL = true; // logs error if boolean is not 0 or 1
private static final boolean FORCE_COLS = true; // ignores row if token count != columns count

private static final String EXPANSION = "Expansion";

public static final int INDEX_NOT_FOUND = -1;

public static TxtParser loadFromFile(FileHandle handle) {
return loadFromStream(handle.read());
}

public static TxtParser loadFromStream(InputStream in) {
BufferedReader reader = null;
try {
reader = IOUtils.buffer(new InputStreamReader(in, "US-ASCII"));
return new TxtParser(reader);
} catch (Throwable t) {
throw new GdxRuntimeException("Couldn't read excel file", t);
}
}

private BufferedReader in;
private ObjectIntMap<String> ids;
private String columns[];

private int index;
private String line;
private String tokens[];

private TxtParser(BufferedReader in) {
this.in = in;

try {
line = in.readLine();
columns = StringUtils.splitPreserveAllTokens(line, '\t');
if (DEBUG_COLS) Gdx.app.debug(TAG, "cols=" + Arrays.toString(columns));

ids = new ObjectIntMap<>();
for (int i = 0; i < columns.length; i++) {
String key = columns[i].toLowerCase();
if (!ids.containsKey(key)) ids.put(key, i);
}
} catch (Throwable t) {
throw new GdxRuntimeException("Couldn't read txt", t);
}
}

@Override
public void close() throws IOException {
if (in == null) {
return;
}

try {
in.close();
} finally {
in = null;
}
}

public String[] getColumns() {
return columns;
}

public int getNumColumns() {
return columns.length;
}

public String getColumnName(int i) {
return columns[i];
}

public String[] getTokens() {
return tokens;
}

public int getNumTokens() {
return tokens.length;
}

public int getColumnId(String s) {
return ids.get(s.toLowerCase(), INDEX_NOT_FOUND);
}

public int[] getColumnId(String[] s) {
int[] columnIds = new int[s.length];
for (int i = 0; i < s.length; i++) columnIds[i] = getColumnId(s[i]);
return columnIds;
}

public int getIndex() {
return index - 1;
}

public String nextLine() {
try {
index++;
line = in.readLine();
if (line == null) {
return null;
} else if (line.equalsIgnoreCase(EXPANSION)) {
return nextLine();
}

tokens = StringUtils.splitPreserveAllTokens(line, '\t');
if (DEBUG_ROWS) Gdx.app.debug(TAG, (index - 1) + ": " + Arrays.toString(tokens));
if (FORCE_COLS && tokens.length != columns.length) {
Gdx.app.error(TAG, "Skipping row " + Arrays.toString(tokens));
return nextLine();
}

return line;
} catch (IOException e) {
Gdx.app.error(TAG, e.getMessage(), e);
return null;
}
}

public String getString(int i) {
if (i == INDEX_NOT_FOUND) return null;
return tokens[i];
}

public byte getByte(int i) {
return NumberUtils.toByte(getString(i));
}

public short getShort(int i) {
return NumberUtils.toShort(getString(i));
}

public int getInt(int i) {
return NumberUtils.toInt(getString(i));
}

public long getLong(int i) {
return NumberUtils.toLong(getString(i));
}

public boolean getBoolean(int i) {
int value = getInt(i);
if (FORCE_BOOL && (value & 1) != value) Gdx.app.error(TAG, String.format("boolean value != 0 or 1 at row %d col %d (\"%s\", \"%s\"): %d", index, i, getString(0), getColumnName(i), value));
return value != 0;
}

public String[] getString(int[] cols) {
String[] data = new String[cols.length];
for (int i = 0; i < cols.length; i++) data[i] = getString(cols[i]);
return data;
}

public byte[] getByte(int[] cols) {
byte[] data = new byte[cols.length];
for (int i = 0; i < cols.length; i++) data[i] = getByte(cols[i]);
return data;
}

public short[] getShort(int[] cols) {
short[] data = new short[cols.length];
for (int i = 0; i < cols.length; i++) data[i] = getShort(cols[i]);
return data;
}

public int[] getInt(int[] cols) {
int[] data = new int[cols.length];
for (int i = 0; i < cols.length; i++) data[i] = getInt(cols[i]);
return data;
}

public long[] getLong(int[] cols) {
long[] data = new long[cols.length];
for (int i = 0; i < cols.length; i++) data[i] = getLong(cols[i]);
return data;
}

public boolean[] getBoolean(int[] cols) {
boolean[] data = new boolean[cols.length];
for (int i = 0; i < cols.length; i++) data[i] = getBoolean(cols[i]);
return data;
}
}

0 comments on commit d2b48b8

Please sign in to comment.
You can’t perform that action at this time.