using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Collections.Concurrent;
using System.Threading.Tasks;
namespace Tango.CSV
{
///
/// Represents a component for reading and writing CSV files.
///
///
public class CsvFile : IDisposable
{
protected bool _is_disposed;
internal protected Stream BaseStream;
protected static DateTime DateTimeZero = new DateTime();
///
/// Initializes the class.
///
static CsvFile()
{
DefaultCsvDefinition = new CsvDefinition
{
EndOfLine = "\r\n",
FieldSeparator = ',',
TextQualifier = '"'
};
UseLambdas = true;
UseTasks = false;
FastIndexOfAny = true;
}
///
/// Gets or sets the default CSV definition.
///
public static CsvDefinition DefaultCsvDefinition { get; set; }
///
/// Gets or sets a value indicating whether [use lambdas].
///
public static bool UseLambdas { get; set; }
///
/// Gets or sets a value indicating whether [use tasks].
///
public static bool UseTasks { get; set; }
///
/// Gets or sets a value indicating whether [fast index of any].
///
public static bool FastIndexOfAny { get; set; }
///
/// Reads the specified CSV source.
///
///
/// The CSV source.
///
public static IEnumerable Read(CsvSource csvSource) where T : new()
{
var csvFileReader = new CsvFileReader(csvSource);
return (IEnumerable)csvFileReader;
}
///
/// Gets the columns.
///
///
/// The CSV source.
///
public static IEnumerable GetColumns(CsvSource csvSource) where T : new()
{
var csvFileReader = new CsvFileReader(csvSource);
return csvFileReader.Columns;
}
///
/// Gets the field separator.
///
///
/// The field separator.
///
public char FieldSeparator { get; private set; }
///
/// Gets the text qualifier.
///
///
/// The text qualifier.
///
public char TextQualifier { get; private set; }
///
/// Gets the columns.
///
///
/// The columns.
///
public IEnumerable Columns { get; protected set; }
///
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
///
public void Dispose()
{
Dispose(true);
}
///
/// Releases unmanaged and - optionally - managed resources.
///
/// true to release both managed and unmanaged resources; false to release only unmanaged resources.
protected virtual void Dispose(bool disposing)
{
// overriden in derived classes
}
}
///
/// Represents a component for reading and writing CSV files from and to a collection of objects.
///
///
///
public class CsvFile : CsvFile
{
private readonly char fieldSeparator;
private readonly string fieldSeparatorAsString;
private readonly char[] invalidCharsInFields;
private readonly StreamWriter streamWriter;
private readonly char textQualifier;
private readonly String[] columns;
private Func[] getters;
readonly bool[] isInvalidCharInFields;
private int linesToWrite;
private readonly BlockingCollection csvLinesToWrite = new BlockingCollection(5000);
private readonly Thread writeCsvLinesTask;
private Task addAsyncTask;
///
/// Initializes a new instance of the class.
///
/// The CSV destination.
public CsvFile(CsvDestination csvDestination)
: this(csvDestination, null)
{
}
///
/// Initializes a new instance of the class.
///
public CsvFile()
{
}
///
/// Initializes a new instance of the class.
///
/// The CSV destination.
/// The CSV definition.
public CsvFile(CsvDestination csvDestination, CsvDefinition csvDefinition)
{
if (csvDefinition == null)
csvDefinition = DefaultCsvDefinition;
this.columns = (csvDefinition.Columns ?? InferColumns(typeof(T))).ToArray();
this.fieldSeparator = csvDefinition.FieldSeparator;
this.fieldSeparatorAsString = this.fieldSeparator.ToString(CultureInfo.InvariantCulture);
this.textQualifier = csvDefinition.TextQualifier;
this.streamWriter = csvDestination.StreamWriter;
this.invalidCharsInFields = new[] { '\r', '\n', this.textQualifier, this.fieldSeparator };
this.isInvalidCharInFields = new bool[256];
foreach (var c in this.invalidCharsInFields)
{
this.isInvalidCharInFields[c] = true;
}
this.WriteHeader();
this.CreateGetters();
if (CsvFile.UseTasks)
{
writeCsvLinesTask = new Thread((o) => this.WriteCsvLines());
writeCsvLinesTask.Start();
}
this.addAsyncTask = Task.Factory.StartNew(() => { });
}
///
/// Releases unmanaged and - optionally - managed resources.
///
/// true to release both managed and unmanaged resources; false to release only unmanaged resources.
protected override void Dispose(bool disposing)
{
if (disposing)
{
_is_disposed = true;
// free managed resources
addAsyncTask.Wait();
if (csvLinesToWrite != null)
{
csvLinesToWrite.CompleteAdding();
}
if (writeCsvLinesTask != null)
writeCsvLinesTask.Join();
this.streamWriter.Close();
}
}
///
/// Infers the columns.
///
/// Type of the record.
///
protected static IEnumerable InferColumns(Type recordType)
{
var columns = recordType
.GetProperties(BindingFlags.Public | BindingFlags.Instance)
.Where(pi => pi.GetIndexParameters().Length == 0
&& pi.GetSetMethod() != null
&& !Attribute.IsDefined(pi, typeof(CsvIgnoreAttribute)))
.Select(pi => pi.Name)
.Concat(recordType
.GetFields(BindingFlags.Public | BindingFlags.Instance)
.Where(fi => !Attribute.IsDefined(fi, typeof(CsvIgnoreAttribute)))
.Select(fi => fi.Name))
.ToList();
return columns;
}
///
/// Writes the CSV lines.
///
private void WriteCsvLines()
{
int written = 0;
foreach (var csvLine in csvLinesToWrite.GetConsumingEnumerable())
{
this.streamWriter.WriteLine(csvLine);
written++;
}
Interlocked.Add(ref this.linesToWrite, -written);
}
///
/// Appends the specified record.
///
/// The record.
public void Append(T record)
{
if (_is_disposed)
{
return;
}
if (CsvFile.UseTasks)
{
var linesWaiting = Interlocked.Increment(ref this.linesToWrite);
Action addRecord = (t) =>
{
if (!_is_disposed)
{
var csvLine = this.ToCsv(record);
this.csvLinesToWrite.Add(csvLine);
}
};
if (linesWaiting < 10000)
this.addAsyncTask = this.addAsyncTask.ContinueWith(addRecord);
else
addRecord(null);
}
else
{
var csvLine = this.ToCsv(record);
this.streamWriter.WriteLine(csvLine);
}
}
///
/// Finds the getter.
///
/// The c.
/// if set to true [static member].
///
private static Func FindGetter(string c, int index, bool staticMember)
{
var flags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.IgnoreCase | (staticMember ? BindingFlags.Static : BindingFlags.Instance);
Func func = null;
PropertyInfo pi = null;
pi = typeof(T).GetProperty(c);
if (pi == null) //Then try get by column index,
{
pi = typeof(T).GetProperties()[index];
pi = typeof(T).GetProperties()[index]; //Workaround for some weired problem that makes the program not take the value.
}
if (CsvFile.UseLambdas)
{
Expression expr = null;
ParameterExpression parameter = Expression.Parameter(typeof(T), "r");
Type type = null;
if (pi != null)
{
type = pi.PropertyType;
expr = Expression.Property(parameter, pi.Name);
}
if (expr != null)
{
Expression> lambda;
if (type.IsValueType)
{
lambda = Expression.Lambda>(Expression.TypeAs(expr, typeof(object)), parameter);
}
else
{
lambda = Expression.Lambda>(expr, parameter);
}
func = lambda.Compile();
}
}
else
{
if (pi != null)
func = o => pi.GetValue(o, null);
}
return func;
}
///
/// Creates the getters.
///
private void CreateGetters()
{
var list = new List>();
for (int i = 0; i < columns.Length; i++)
{
Func func = null;
var propertyName = (columns[i].IndexOf(' ') < 0 ? columns[i] : columns[i].Replace(" ", ""));
func = FindGetter(columns[i], i, false) ?? FindGetter(columns[i], i, true);
list.Add(func);
}
this.getters = list.ToArray();
}
///
/// To the CSV.
///
/// The record.
///
/// Cannot be null;record
private string ToCsv(T record)
{
if (record == null)
throw new ArgumentException("Cannot be null", "record");
string[] csvStrings = new string[getters.Length];
for (int i = 0; i < getters.Length; i++)
{
var getter = getters[i];
object fieldValue = getter == null ? null : getter(record);
csvStrings[i] = this.ToCsvString(fieldValue);
}
return string.Join(this.fieldSeparatorAsString, csvStrings);
}
///
/// To the CSV string.
///
/// The o.
///
private string ToCsvString(object o)
{
if (o != null)
{
string valueString = o as string ?? Convert.ToString(o, CultureInfo.CurrentUICulture);
if (RequireQuotes(valueString))
{
var csvLine = new StringBuilder();
csvLine.Append(this.textQualifier);
foreach (char c in valueString)
{
if (c == this.textQualifier)
csvLine.Append(c); // double the double quotes
csvLine.Append(c);
}
csvLine.Append(this.textQualifier);
return csvLine.ToString();
}
else
return valueString;
}
return string.Empty;
}
///
/// Requires the quotes.
///
/// The value string.
///
private bool RequireQuotes(string valueString)
{
if (CsvFile.FastIndexOfAny)
{
var len = valueString.Length;
for (int i = 0; i < len; i++)
{
char c = valueString[i];
if (c <= 255 && this.isInvalidCharInFields[c])
return true;
}
return false;
}
else
{
return valueString.IndexOfAny(this.invalidCharsInFields) >= 0;
}
}
///
/// Writes the header.
///
private void WriteHeader()
{
var csvLine = new StringBuilder();
for (int i = 0; i < this.columns.Length; i++)
{
if (i > 0)
csvLine.Append(this.fieldSeparator);
csvLine.Append(this.ToCsvString(this.columns[i]));
}
this.streamWriter.WriteLine(csvLine.ToString());
}
}
}