[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[coldsync-hackers] MobileDB to/from CSV converter

To: coldsync-hackers@lusars.net
Subject: [coldsync-hackers] MobileDB to/from CSV converter
From: flint@kiva.net
Date: Fri, 20 Jul 2001 14:56:56 -0500
Delivered-To: coldsync-hackers-outgoing@rocky.lusars.net
Reply-To: coldsync-hackers@lusars.net
Sender: owner-coldsync-hackers@rocky.lusars.net

	I'm posting this mostly for suggestions.  It uses the following modules:

* the p5-Palm modules from the coldsync page
* the Text::CSV module from CPAN
* File::Basename, FileHandle, Getopt::Std, and Storable are standard with 
Perl

	I guess the coolest thing would be a CSV/MobileDB conduit, but right now 
this seems to convert small MobileDB files to text files with 
comma-separated values, and back.  The file 'blank.pdb' is a mostly-empty 
MobileDB file that I created on my Palm and transferred over.  I would like 
to know how to translate CSV to MobileDB without having such a "starter" 
file.  I'm reading from the source file into memory, then writing the data 
out in the destination file -- this could obviously burn lots of memory in 
the case of a large database.  But the advantage is that something else 
could be substituted for CSV.

# Palm::MobileDB.pm
#
# Perl class for dealing with MobileDB files.
#
# Based on Palm::Memo by Andrew Arensburger.

use strict;
package Palm::MobileDB;
use Palm::Raw();
use Palm::StdAppInfo();
use vars qw( $VERSION @ISA );

$VERSION = sprintf "%d.%03d", '$Revision: 1.10 $ ' =~ m{(\d+)\.(\d+)};
@ISA = qw( Palm::Raw Palm::StdAppInfo );

=head1 NAME

Palm::Memo - Handler for Palm Memo databases.

=head1 SYNOPSIS

    use Palm::Memo;

=head1 DESCRIPTION

The Memo PDB handler is a helper class for the Palm::PDB package. It
parses Memo databases.

=head2 AppInfo block

The AppInfo block begins with standard category support. See
L<Palm::StdAppInfo> for details.

Other fields include:

    $pdb->{appinfo}{sortOrder}

I don't know what this is.

=head2 Sort block

    $pdb->{sort}

This is a scalar, the raw data of the sort block.

=head2 Records

    $record = $pdb->{records}[N]

    $record->{data}

A string, the text of the memo.

=cut
#'

sub import
{
	&Palm::PDB::RegisterPDBHandlers(__PACKAGE__,
		[ "Mdb1", "Mdb1" ],
		);
}

=head2 new

  $pdb = new Palm::Memo;

Create a new PDB, initialized with the various Palm::Memo fields
and an empty record list.

Use this method if you're creating a Memo PDB from scratch.

=cut
#'
sub new
{
	my $classname	= shift;
	my $self	= $classname->SUPER::new(@_);
			# Create a generic PDB. No need to rebless it,
			# though.

	$self->{name} = "MobileDB";	# Default
	$self->{creator} = "Mdb1";
	$self->{type} = "Mdb1";
	$self->{attributes}{resource} = 0;
				# The PDB is not a resource database by
				# default, but it's worth emphasizing,
				# since MemoDB is explicitly not a PRC.

	# Initialize the AppInfo block
	$self->{appinfo} = {
		sortOrder	=> undef,	# XXX - ?
	};

	# Add the standard AppInfo block stuff
	&Palm::StdAppInfo::seed_StdAppInfo($self->{appinfo});

	# Give the PDB a blank sort block
	$self->{sort} = undef;

	# Give the PDB an empty list of records
	$self->{records} = [];

	return $self;
}

=head2 new_Record

  $record = $pdb->new_Record;

Creates a new Memo record, with blank values for all of the fields.

C<new_Record> does B<not> add the new record to C<$pdb>. For that,
you want C<$pdb-E<gt>append_Record>.

=cut

sub new_Record
{
	my $classname = shift;
	my $retval = $classname->SUPER::new_Record(@_);

	$retval->{data} = "";

	return $retval;
}

# ParseAppInfoBlock
# Parse the AppInfo block for Memo databases.
sub ParseAppInfoBlock
{
	my $self = shift;
	my $data = shift;
	my $sortOrder;
	my $i;
	my $appinfo = {};
	my $std_len;

	# Get the standard parts of the AppInfo block
	$std_len = &Palm::StdAppInfo::parse_StdAppInfo($appinfo, $data);

	$data = $appinfo->{other};		# Look at the non-category part

	# Get the rest of the AppInfo block
	my $unpackstr =		# Argument to unpack()
		"x4" .		# Padding
		"C";		# Sort order

	($sortOrder) = unpack $unpackstr, $data;

	$appinfo->{sortOrder} = $sortOrder;

	return $appinfo;
}

sub PackAppInfoBlock
{
	my $self = shift;
	my $retval;
	my $i;

	# Pack the non-category part of the AppInfo block
	$self->{appinfo}{other} =
		pack("x4 C x1", $self->{appinfo}{sortOrder});

	# Pack the AppInfo block
	$retval = &Palm::StdAppInfo::pack_StdAppInfo($self->{appinfo});

	return $retval;
}

sub PackSortBlock
{
	# XXX
	return undef;
}

sub ParseRecord
{
	my $self = shift;
	my %record = @_;

	delete $record{offset};		# This is useless
	$record{data} =~ s/^\xff\xff\xff\x01\xff\x00//; # Trim record head
	$record{data} =~ s/\x00\xff$//;	# Trim record tail

	return \%record;
}

sub PackRecord
{
	my $self = shift;
	my $record = shift;

	return "\xff\xff\xff\x01\xff\x00".$record->{data}."\x00\xff";
}

1;
__END__

=head1 AUTHOR

Andrew Arensburger E<lt>arensb@ooblick.comE<gt>

=head1 SEE ALSO

Palm::PDB(3)

Palm::StdAppInfo(3)

=cut

#!/usr/bin/perl -w

# tpmdbconv.pl

# Tom's Perl MobileDB Converter -- converts CSV files to/from MobileDB files
# By Tom Lee <tjlee@mh5.com>
# Begun 2001/07/18

# This is based on Adam Glass's 'pmdbconv'

# No warranties expressed or implied; use this code at your own risk

# Released under the GNU Public License (GPL)

use File::Basename;
use FileHandle;
use Getopt::Std;
use lib qw(.);
use Palm::PDB;
use Palm::MobileDB;
use strict;
use Storable;
use Text::CSV;
use vars qw($opt_v $opt_i $opt_o $opt_n);

# Generic appinfo block for MobileDB

my(%app_info_block) =
  (
   other => '',
   lastUniqueID => 15,
   categories => [
		    {
		     renamed => 0,
		     id => 0,
		     name => 'Unfiled'
		    },
		    {
		     renamed => 0,
		     id => 1,
		     name => 'FieldLabels'
		    },
		    {
		     renamed => 0,
		     id => 2,
		     name => 'DataRecords'
		    },
		    {
		     renamed => 0,
		     id => 3,
		     name => 'DataRecordsFout'
		    },
		    {
		     renamed => 0,
		     id => 4,
		     name => 'Preferences'
		    },
		    {
		     renamed => 0,
		     id => 5,
		     name => 'DataType'
		    },
		    {
		     renamed => 0,
		     id => 6,
		     name => 'FieldLengths'
		    },
		    {
		     renamed => 0,
		     id => 7,
		     name => ''
		    },
		    {
		     renamed => 0,
		     id => 8,
		     name => ''
		    },
		    {
		     renamed => 0,
		     id => 9,
		     name => ''
		    },
		    {
		     renamed => 0,
		     id => 10,
		     name => ''
		    },
		    {
		     renamed => 0,
		     id => 11,
		     name => ''
		    },
		    {
		     renamed => 0,
		     id => 12,
		     name => ''
		    },
		    {
		     renamed => 0,
		     id => 13,
		     name => ''
		    },
		    {
		     renamed => 0,
		     id => 14,
		     name => ''
		    },
		    {
		     renamed => 0,
		     id => 15,
		     name => ''
		    }
		   ],
   sortOrder => 0,
  );

sub Repeat {
  my($element, $number) = @_;
  my(@array) = ();
  for(my($i) = 0; $i < $number; $i++) {
    push(@array, $element);
  }
  return(\@array);
}

sub PackData {
  my($data) = @_;

  my($i) = 0;
  return(join('', map {
    sprintf("\x00%s%s", chr($i++), $_);
  } @$data));
}

# Generic field type data record for MobileDB

my($type_data) = &PackData(&Repeat('str', 20));

# Generic field length data record for MobileDB

my($length_data) = &PackData(&Repeat('80', 20));

sub ReadCSV {
  my($fh) = @_;
  my($csv) = new Text::CSV();

  # Bins to toss data in

  my(@headers) = ();
  my(@data) = ();

  # Get the header line

  if(defined(my $line = $fh->getline())) {

    # Parse the header line

    if($csv->parse($line)) {
      @headers = $csv->fields();

      # Go through the data

      while(defined($line = $fh->getline())) {
	if($csv->parse($line)) {

	  # Set the data up as a hash; the header line gives the order

	  my(@record) = $csv->fields();
	  my(%hash) = map {
	    $_ => shift(@record);
	  } @headers;

	  # @data is an array of hashrefs

	  push(@data, \%hash);
	} else {
	  die("Unable to parse line of data\n");
	}
      }
    } else {
      die("Unable to parse first line\n");
    }
  } else {
    die("Unable to read first line\n");
  }
  return(\@headers, \@data);
}

sub WriteCSV {
  my($fh, $headers, $data) = @_;
  my($csv) = new Text::CSV();
  if($csv->combine(@$headers)) {
    printf($fh "%s\n", $csv->string());
    foreach my $href (@$data) {
      my(@record) = map {
	$href->{$_};
      } @$headers;
      if($csv->combine(@record)) {
	printf($fh "%s\n", $csv->string());
      } else {
	die("Unable to write line of data for some reason\n");
      }
    }
  } else {
    die("Unable to write header line for some reason\n");
  }
}

sub ParsePDB {
  my($pdb) = @_;
  my(@pdb_records) = @{$pdb->{records}};
  my(@records) = ();

  # Split records by categories.  1 = headers, 2 and 3 are data (3
  # means filtered out, but that's not supported by CSV)

  my(@header_records) = grep {
    $_->{category} == 1;
  } @pdb_records;
  my(@data_records) = grep {
    ($_->{category} == 2) || ($_->{category} == 3);
  } @pdb_records;

  # There should be exactly one record with category 1.

  my($header_data) = $header_records[0]->{data};
  my(@headers) = ($header_data =~ /\x00.([^\x00]+)/g);
  foreach my $data_record (@data_records) {
    my($record_data) = $data_record->{data};
    my(@data) = ($record_data =~ /\x00.([^\x00]+)/g);
    my(%hash) = map {
      $_ => shift(@data);
    } @headers;
    push(@records, \%hash);
  }
  return(\@headers, \@records);
}

sub ConstructPDB {
  my($pdb, $headers, $records) = @_;

  # Clear out any previous data.

  $pdb->{records} = [];

  # Make the headers record.

  my($pdb_record) = new_Record Palm::PDB();
  $pdb_record->{attributes}->{archive} = 1;
  $pdb_record->{category} = 1;
  $pdb_record->{data} = &PackData($headers);
  $pdb->append_Record($pdb_record);

  # Make type and length records.

  $pdb_record = new_Record Palm::PDB();
  $pdb_record->{attributes}->{archive} = 1;
  $pdb_record->{category} = 5;
  $pdb_record->{data} = $type_data;
  $pdb->append_Record($pdb_record);

  $pdb_record = new_Record Palm::PDB();
  $pdb_record->{attributes}->{archive} = 1;
  $pdb_record->{category} = 6;
  $pdb_record->{data} = $length_data;
  $pdb->append_Record($pdb_record);

  # Add the data.

  foreach my $record (@$records) {
    my($pdb_record) = new_Record Palm::PDB();
    $pdb_record->{attributes}->{archive} = 1;
    $pdb_record->{category} = 2;
    $pdb_record->{data} = &PackData([ map {
      $record->{$_};
    } @$headers ]);
    $pdb->append_Record($pdb_record);
  }
  $pdb->{name} = $opt_n || 'NamelessDB';
  $pdb->{attributes}->{backup} = 1;
  $pdb->{type} = 'Mdb1';
  $pdb->{creator} = 'Mdb1';
  $pdb->{appinfo} = &Storable::dclone(\%app_info_block);
  $pdb->{version} = 0;
  $pdb->{modnum} = 0;
}

###############################################################################
# Main program
###############################################################################

getopts('vi:o:n:');

my($i_type, $o_type);
(undef, undef, $i_type) = fileparse($opt_i, '\..*');
(undef, undef, $o_type) = fileparse($opt_o, '\..*');

if($i_type eq $o_type) {
  die("Input and output file extensions identical -- use cp\n");
}

# Read input file

my(@headers) = ();
my(@records) = ();
if($i_type eq '.pdb') {
  my($pdb) = new Palm::PDB();
  if($pdb->Load($opt_i)) {
    my($h, $r) = &ParsePDB($pdb);
    @headers = @$h;
    @records = @$r;
  } else {
    die("Unable to open $opt_i: $!\n");
  }
} else {
  my($fh) = new FileHandle;
  if($fh->open("<$opt_i")) {
    my($h, $r) = &ReadCSV($fh);
    $fh->close();
    @headers = @$h;
    @records = @$r;
  } else {
    die("Unable to open $opt_i: $!\n");
  }
}

# Write output file

if($o_type eq '.pdb') {
  my($pdb) = new Palm::PDB();
  $pdb->Load('blank.pdb');
#  use Data::Dumper;
#  warn(Dumper($pdb));
  &ConstructPDB($pdb, \@headers, \@records);
  unless($pdb->Write($opt_o)) {
    die("Unable to open $opt_o: $!\n");
  }
} else {
  my($fh) = new FileHandle;
  if($fh->open(">$opt_o")) {
    &WriteCSV($fh, \@headers, \@records);
    $fh->close();
  } else {
    die("Unable to open $opt_o: $!\n");
  }
}

Take care,
Tom Lee (flint@kiva.net)

Prev by Date: Re: [coldsync-hackers] Visor Edge not recognized
Next by Date: Re: [coldsync-hackers] Patch: Palm m50x USB support (Linux)
Previous by thread: Re: [coldsync-hackers] Visor Edge not recognized
Next by thread: [coldsync-hackers] Delete Database in a Conduit?
Index(es):
- Date
- Thread