1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
/* Generated by the protocol buffer compiler. DO NOT EDIT! */
/* Generated from: CreateRequest.proto */
#ifndef PROTOBUF_C_CreateRequest_2eproto__INCLUDED
#define PROTOBUF_C_CreateRequest_2eproto__INCLUDED
#include <protobuf-c/protobuf-c.h>
PROTOBUF_C__BEGIN_DECLS
#if PROTOBUF_C_VERSION_NUMBER < 1003000
# error This file was generated by a newer version of protoc-c which is incompatible with your libprotobuf-c headers. Please update your headers.
#elif 1003000 < PROTOBUF_C_MIN_COMPILER_VERSION
# error This file was generated by an older version of protoc-c which is incompatible with your libprotobuf-c headers. Please regenerate this file with a newer version of protoc-c.
#endif
#include "FileAttribute.pb-c.h"
typedef struct _CreateRequest CreateRequest;
/* --- enums --- */
/* --- messages --- */
struct _CreateRequest
{
ProtobufCMessage base;
protobuf_c_boolean has_attribute;
FileAttribute attribute;
char *path;
};
#define CREATE_REQUEST__INIT \
{ PROTOBUF_C_MESSAGE_INIT (&create_request__descriptor) \
, 0, FILE_ATTRIBUTE__Unspecified, NULL }
/* CreateRequest methods */
void create_request__init
(CreateRequest *message);
size_t create_request__get_packed_size
(const CreateRequest *message);
size_t create_request__pack
(const CreateRequest *message,
uint8_t *out);
size_t create_request__pack_to_buffer
(const CreateRequest *message,
ProtobufCBuffer *buffer);
CreateRequest *
create_request__unpack
(ProtobufCAllocator *allocator,
size_t len,
const uint8_t *data);
void create_request__free_unpacked
(CreateRequest *message,
ProtobufCAllocator *allocator);
/* --- per-message closures --- */
typedef void (*CreateRequest_Closure)
(const CreateRequest *message,
void *closure_data);
/* --- services --- */
/* --- descriptors --- */
extern const ProtobufCMessageDescriptor create_request__descriptor;
PROTOBUF_C__END_DECLS
#endif /* PROTOBUF_Cusing System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Collections.Concurrent;
using System.Threading.Tasks;
namespace Tango.CSV
{
/// <summary>
/// Represents a component for reading and writing CSV files.
/// </summary>
/// <seealso cref="System.IDisposable" />
public class CsvFile : IDisposable
{
protected bool _is_disposed;
internal protected Stream BaseStream;
protected static DateTime DateTimeZero = new DateTime();
/// <summary>
/// Initializes the <see cref="CsvFile"/> class.
/// </summary>
static CsvFile()
{
DefaultCsvDefinition = new CsvDefinition
{
EndOfLine = "\r\n",
FieldSeparator = ',',
TextQualifier = '"'
};
UseLambdas = true;
UseTasks = false;
FastIndexOfAny = true;
}
/// <summary>
/// Gets or sets the default CSV definition.
/// </summary>
public static CsvDefinition DefaultCsvDefinition { get; set; }
/// <summary>
/// Gets or sets a value indicating whether [use lambdas].
/// </summary>
public static bool UseLambdas { get; set; }
/// <summary>
/// Gets or sets a value indicating whether [use tasks].
/// </summary>
public static bool UseTasks { get; set; }
/// <summary>
/// Gets or sets a value indicating whether [fast index of any].
/// </summary>
public static bool FastIndexOfAny { get; set; }
/// <summary>
/// Reads the specified CSV source.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="csvSource">The CSV source.</param>
/// <returns></returns>
public static IEnumerable<T> Read<T>(CsvSource csvSource) where T : new()
{
var csvFileReader = new CsvFileReader<T>(csvSource);
return (IEnumerable<T>)csvFileReader;
}
/// <summary>
/// Gets the columns.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="csvSource">The CSV source.</param>
/// <returns></returns>
public static IEnumerable<string> GetColumns<T>(CsvSource csvSource) where T : new()
{
var csvFileReader = new CsvFileReader<T>(csvSource);
return csvFileReader.Columns;
}
/// <summary>
/// Gets the field separator.
/// </summary>
/// <value>
/// The field separator.
/// </value>
public char FieldSeparator { get; private set; }
/// <summary>
/// Gets the text qualifier.
/// </summary>
/// <value>
/// The text qualifier.
/// </value>
public char TextQualifier { get; private set; }
/// <summary>
/// Gets the columns.
/// </summary>
/// <value>
/// The columns.
/// </value>
public IEnumerable<String> Columns { get; protected set; }
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
public void Dispose()
{
Dispose(true);
}
/// <summary>
/// Releases unmanaged and - optionally - managed resources.
/// </summary>
/// <param name="disposing"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
protected virtual void Dispose(bool disposing)
{
// overriden in derived classes
}
}
/// <summary>
/// Represents a component for reading and writing CSV files from and to a collection of objects.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <seealso cref="System.IDisposable" />
public class CsvFile<T> : CsvFile
{
private readonly char fieldSeparator;
private readonly string fieldSeparatorAsString;
private readonly char[] invalidCharsInFields;
private readonly StreamWriter streamWriter;
private readonly char textQualifier;
private readonly String[] columns;
private Func<T, object>[] getters;
readonly bool[] isInvalidCharInFields;
private int linesToWrite;
private readonly BlockingCollection<string> csvLinesToWrite = new BlockingCollection<string>(5000);
private readonly Thread writeCsvLinesTask;
private Task addAsyncTask;
/// <summary>
/// Initializes a new instance of the <see cref="CsvFile{T}"/> class.
/// </summary>
/// <param name="csvDestination">The CSV destination.</param>
public CsvFile(CsvDestination csvDestination)
: this(csvDestination, null)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="CsvFile{T}"/> class.
/// </summary>
public CsvFile()
{
}
/// <summary>
/// Initializes a new instance of the <see cref="CsvFile{T}"/> class.
/// </summary>
/// <param name="csvDestination">The CSV destination.</param>
/// <param name="csvDefinition">The CSV definition.</param>
public CsvFile(CsvDestination csvDestination, CsvDefinition csvDefinition)
{
if (csvDefinition == null)
csvDefinition = DefaultCsvDefinition;
this.columns = (csvDefinition.Columns ?? InferColumns(typeof(T))).ToArray();
this.fieldSeparator = csvDefinition.FieldSeparator;
this.fieldSeparatorAsString = this.fieldSeparator.ToString(CultureInfo.InvariantCulture);
this.textQualifier = csvDefinition.TextQualifier;
this.streamWriter = csvDestination.StreamWriter;
this.invalidCharsInFields = new[] { '\r', '\n', this.textQualifier, this.fieldSeparator };
this.isInvalidCharInFields = new bool[256];
foreach (var c in this.invalidCharsInFields)
{
this.isInvalidCharInFields[c] = true;
}
this.WriteHeader();
this.CreateGetters();
if (CsvFile.UseTasks)
{
writeCsvLinesTask = new Thread((o) => this.WriteCsvLines());
writeCsvLinesTask.Start();
}
this.addAsyncTask = Task.Factory.StartNew(() => { });
}
/// <summary>
/// Releases unmanaged and - optionally - managed resources.
/// </summary>
/// <param name="disposing"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (disposing)
{
_is_disposed = true;
// free managed resources
addAsyncTask.Wait();
if (csvLinesToWrite != null)
{
csvLinesToWrite.CompleteAdding();
}
if (writeCsvLinesTask != null)
writeCsvLinesTask.Join();
this.streamWriter.Close();
}
}
/// <summary>
/// Infers the columns.
/// </summary>
/// <param name="recordType">Type of the record.</param>
/// <returns></returns>
protected static IEnumerable<string> InferColumns(Type recordType)
{
var columns = recordType
.GetProperties(BindingFlags.Public | BindingFlags.Instance)
.Where(pi => pi.GetIndexParameters().Length == 0
&& pi.GetSetMethod() != null
&& !Attribute.IsDefined(pi, typeof(CsvIgnoreAttribute)))
.Select(pi => pi.Name)
.Concat(recordType
.GetFields(BindingFlags.Public | BindingFlags.Instance)
.Where(fi => !Attribute.IsDefined(fi, typeof(CsvIgnoreAttribute)))
.Select(fi => fi.Name))
.ToList();
return columns;
}
/// <summary>
/// Writes the CSV lines.
/// </summary>
private void WriteCsvLines()
{
int written = 0;
foreach (var csvLine in csvLinesToWrite.GetConsumingEnumerable())
{
this.streamWriter.WriteLine(csvLine);
written++;
}
Interlocked.Add(ref this.linesToWrite, -written);
}
/// <summary>
/// Appends the specified record.
/// </summary>
/// <param name="record">The record.</param>
public void Append(T record)
{
if (_is_disposed)
{
return;
}
if (CsvFile.UseTasks)
{
var linesWaiting = Interlocked.Increment(ref this.linesToWrite);
Action<Task> addRecord = (t) =>
{
if (!_is_disposed)
{
var csvLine = this.ToCsv(record);
this.csvLinesToWrite.Add(csvLine);
}
};
if (linesWaiting < 10000)
this.addAsyncTask = this.addAsyncTask.ContinueWith(addRecord);
else
addRecord(null);
}
else
{
var csvLine = this.ToCsv(record);
this.streamWriter.WriteLine(csvLine);
}
}
/// <summary>
/// Finds the getter.
/// </summary>
/// <param name="c">The c.</param>
/// <param name="staticMember">if set to <c>true</c> [static member].</param>
/// <returns></returns>
private static Func<T, object> FindGetter(string c, int index, bool staticMember)
{
var flags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.IgnoreCase | (staticMember ? BindingFlags.Static : BindingFlags.Instance);
Func<T, object> func = null;
PropertyInfo pi = null;
pi = typeof(T).GetProperty(c);
if (pi == null) //Then try get by column index,
{
pi = typeof(T).GetProperties()[index];
pi = typeof(T).GetProperties()[index]; //Workaround for some weired problem that makes the program not take the value.
}
if (CsvFile.UseLambdas)
{
Expression expr = null;
ParameterExpression parameter = Expression.Parameter(typeof(T), "r");
Type type = null;
if (pi != null)
{
type = pi.PropertyType;
expr = Expression.Property(parameter, pi.Name);
}
if (expr != null)
{
Expression<Func<T, object>> lambda;
if (type.IsValueType)
{
lambda = Expression.Lambda<Func<T, object>>(Expression.TypeAs(expr, typeof(object)), parameter);
}
else
{
lambda = Expression.Lambda<Func<T, object>>(expr, parameter);
}
func = lambda.Compile();
}
}
else
{
if (pi != null)
func = o => pi.GetValue(o, null);
}
return func;
}
/// <summary>
/// Creates the getters.
/// </summary>
private void CreateGetters()
{
var list = new List<Func<T, object>>();
for (int i = 0; i < columns.Length; i++)
{
Func<T, Object> func = null;
var propertyName = (columns[i].IndexOf(' ') < 0 ? columns[i] : columns[i].Replace(" ", ""));
func = FindGetter(columns[i], i, false) ?? FindGetter(columns[i], i, true);
list.Add(func);
}
this.getters = list.ToArray();
}
/// <summary>
/// To the CSV.
/// </summary>
/// <param name="record">The record.</param>
/// <returns></returns>
/// <exception cref="System.ArgumentException">Cannot be null;record</exception>
private string ToCsv(T record)
{
if (record == null)
throw new ArgumentException("Cannot be null", "record");
string[] csvStrings = new string[getters.Length];
for (int i = 0; i < getters.Length; i++)
{
var getter = getters[i];
object fieldValue = getter == null ? null : getter(record);
csvStrings[i] = this.ToCsvString(fieldValue);
}
return string.Join(this.fieldSeparatorAsString, csvStrings);
}
/// <summary>
/// To the CSV string.
/// </summary>
/// <param name="o">The o.</param>
/// <returns></returns>
private string ToCsvString(object o)
{
if (o != null)
{
string valueString = o as string ?? Convert.ToString(o, CultureInfo.CurrentUICulture);
if (RequireQuotes(valueString))
{
var csvLine = new StringBuilder();
csvLine.Append(this.textQualifier);
foreach (char c in valueString)
{
if (c == this.textQualifier)
csvLine.Append(c); // double the double quotes
csvLine.Append(c);
}
csvLine.Append(this.textQualifier);
return csvLine.ToString();
}
else
return valueString;
}
return string.Empty;
}
/// <summary>
/// Requires the quotes.
/// </summary>
/// <param name="valueString">The value string.</param>
/// <returns></returns>
private bool RequireQuotes(string valueString)
{
if (CsvFile.FastIndexOfAny)
{
var len = valueString.Length;
for (int i = 0; i < len; i++)
{
char c = valueString[i];
if (c <= 255 && this.isInvalidCharInFields[c])
return true;
}
return false;
}
else
{
return valueString.IndexOfAny(this.invalidCharsInFields) >= 0;
}
}
/// <summary>
/// Writes the header.
/// </summary>
private void WriteHeader()
{
var csvLine = new StringBuilder();
for (int i = 0; i < this.columns.Length; i++)
{
if (i > 0)
csvLine.Append(this.fieldSeparator);
csvLine.Append(this.ToCsvString(this.columns[i]));
}
this.streamWriter.WriteLine(csvLine.ToString());
}
}
}
|
