+++ /dev/null
-#!/usr/bin/perl -w
-# unicode-muncher.pl -- generate Unicode database for java.lang.Character
-# Copyright (C) 1998, 2002 Free Software Foundation, Inc.
-#
-# This file is part of GNU Classpath.
-#
-# GNU Classpath is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# GNU Classpath is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with GNU Classpath; see the file COPYING. If not, write to the
-# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# 02111-1307 USA.
-#
-# Linking this library statically or dynamically with other modules is
-# making a combined work based on this library. Thus, the terms and
-# conditions of the GNU General Public License cover the whole
-# combination.
-#
-# As a special exception, the copyright holders of this library give you
-# permission to link this library with independent modules to produce an
-# executable, regardless of the license terms of these independent
-# modules, and to copy and distribute the resulting executable under
-# terms of your choice, provided that you also meet, for each linked
-# independent module, the terms and conditions of the license of that
-# module. An independent module is a module which is not derived from
-# or based on this library. If you modify this library, you may extend
-# this exception to your version of the library, but you are not
-# obligated to do so. If you do not wish to do so, delete this
-# exception statement from your version.
-
-# Code for reading UnicodeData.txt and generating the code for
-# gnu.java.lang.CharData. For now, the relevant Unicode definition files
-# are found in libjava/gnu/gcj/convert/.
-#
-# Inspired by code from Jochen Hoenicke.
-# author Eric Blake <ebb9@email.byu.edu>
-#
-# Usage: ./unicode-muncher <UnicodeData.txt> <CharData.java>
-# where <UnicodeData.txt> is obtained from www.unicode.org (named
-# UnicodeData-3.0.0.txt for Unicode version 3.0.0), and <CharData.java>
-# is the final location for the Java interface gnu.java.lang.CharData.
-# As of JDK 1.4, use Unicode version 3.0.0 for best results.
-
-##
-## Convert a 16-bit integer to a Java source code String literal character
-##
-sub javaChar($) {
- my ($char) = @_;
- die "Out of range: $char\n" if $char < -0x8000 or $char > 0xffff;
- $char += 0x10000 if $char < 0;
- # Special case characters that must be escaped, or are shorter as ASCII
- return sprintf("\\%03o", $char) if $char < 0x20;
- return "\\\"" if $char == 0x22;
- return "\\\\" if $char == 0x5c;
- return pack("C", $char) if $char < 0x7f;
- return sprintf("\\u%04x", $char);
-}
-
-##
-## Convert the text UnicodeData file from www.unicode.org into a Java
-## interface with string constants holding the compressed information.
-##
-my @TYPECODES = qw(Cn Lu Ll Lt Lm Lo Mn Me Mc Nd Nl No Zs Zl Zp Cc Cf
- SKIPPED Co Cs Pd Ps Pe Pc Po Sm Sc Sk So Pi Pf);
-my @DIRCODES = qw(L R AL EN ES ET AN CS NSM BN B S WS ON LRE LRO RLE RLO PDF);
-
-my $NOBREAK_FLAG = 32;
-my $MIRRORED_FLAG = 64;
-
-my @info = ();
-my $titlecase = "";
-my $count = 0;
-my $range = 0;
-
-die "Usage: $0 <UnicodeData.txt> <CharData.java>" unless @ARGV == 2;
-open (UNICODE, "< $ARGV[0]") || die "Can't open Unicode attribute file: $!\n";
-
-# Stage 1: Parse the attribute file
-$| = 1;
-print "GNU Classpath Unicode Attribute Database Generator 2.0\n";
-print "Copyright (C) 1998, 2002 Free Software Foundation, Inc.\n";
-print "Parsing attributes file";
-while(<UNICODE>) {
- print "." unless $count++ % 1000;
- chomp;
- s/\r//g;
- my ($ch, $name, $category, undef, $bidir, $decomp, undef, undef, $numeric,
- $mirrored, undef, undef, $upcase, $lowcase, $title) = split ';';
- $ch = hex($ch);
- next if $ch > 0xffff; # Ignore surrogate pairs, since Java does
-
- my ($type, $numValue, $upperchar, $lowerchar, $direction);
-
- $type = 0;
- while ($category !~ /^$TYPECODES[$type]$/) {
- if (++$type == @TYPECODES) {
- die "$ch: Unknown type: $category";
- }
- }
- $type |= $NOBREAK_FLAG if ($decomp =~ /noBreak/);
- $type |= $MIRRORED_FLAG if ($mirrored =~ /Y/);
-
- if ($numeric =~ /^[0-9]+$/) {
- $numValue = $numeric;
- die "numValue too big: $ch, $numValue\n" if $numValue >= 0x7fff;
- } elsif ($numeric eq "") {
- # Special case sequences of 'a'-'z'
- if ($ch >= 0x0041 && $ch <= 0x005a) {
- $numValue = $ch - 0x0037;
- } elsif ($ch >= 0x0061 && $ch <= 0x007a) {
- $numValue = $ch - 0x0057;
- } elsif ($ch >= 0xff21 && $ch <= 0xff3a) {
- $numValue = $ch - 0xff17;
- } elsif ($ch >= 0xff41 && $ch <= 0xff5a) {
- $numValue = $ch - 0xff37;
- } else {
- $numValue = -1;
- }
- } else {
- $numValue = -2;
- }
-
- $upperchar = $upcase ? hex($upcase) - $ch : 0;
- $lowerchar = $lowcase ? hex($lowcase) - $ch : 0;
- if ($title ne $upcase) {
- my $titlechar = $title ? hex($title) : $ch;
- $titlecase .= pack("n2", $ch, $titlechar);
- }
-
- $direction = 0;
- while ($bidir !~ /^$DIRCODES[$direction]$/) {
- if (++$direction == @DIRCODES) {
- $direction = -1;
- last;
- }
- }
-
- if ($range) {
- die "Expecting end of range at $ch\n" unless $name =~ /Last>$/;
- for ($range + 1 .. $ch - 1) {
- $info[$_] = pack("n5", $type, $numValue, $upperchar,
- $lowerchar, $direction);
- }
- $range = 0;
- } elsif ($name =~ /First>$/) {
- $range = $ch;
- }
- $info[$ch] = pack("n5", $type, $numValue, $upperchar, $lowerchar,
- $direction);
-}
-close UNICODE;
-
-# Stage 2: Compress the data structures
-printf "\nCompressing data structures";
-$count = 0;
-my $info = ();
-my %charhash = ();
-my @charinfo = ();
-
-for my $ch (0 .. 0xffff) {
- print "." unless $count++ % 0x1000;
- if (! defined $info[$ch]) {
- $info[$ch] = pack("n5", 0, -1, 0, 0, -1);
- }
-
- my ($type, $numVal, $upper, $lower, $direction) = unpack("n5", $info[$ch]);
- if (! exists $charhash{$info[$ch]}) {
- push @charinfo, [ $numVal, $upper, $lower, $direction ];
- $charhash{$info[$ch]} = $#charinfo;
- }
- $info .= pack("n", ($charhash{$info[$ch]} << 7) | $type);
-}
-
-my $charlen = @charinfo;
-my $bestshift;
-my $bestest = 1000000;
-my $bestblkstr;
-die "Too many unique character entries: $charlen\n" if $charlen > 512;
-print "\nUnique character entries: $charlen\n";
-
-for my $i (3 .. 8) {
- my $blksize = 1 << $i;
- my %blocks = ();
- my @blkarray = ();
- my ($j, $k);
- print "shift: $i";
-
- for ($j = 0; $j < 0x10000; $j += $blksize) {
- my $blkkey = substr $info, 2 * $j, 2 * $blksize;
- if (! exists $blocks{$blkkey}) {
- push @blkarray, $blkkey;
- $blocks{$blkkey} = $#blkarray;
- }
- }
- my $blknum = @blkarray;
- my $blocklen = $blknum * $blksize;
- printf " before %5d", $blocklen;
-
- # Now we try to pack the blkarray as tight as possible by finding matching
- # heads and tails.
- for ($j = $blksize - 1; $j > 0; $j--) {
- my %tails = ();
- for $k (0 .. $#blkarray) {
- next if ! defined $blkarray[$k];
- my $len = length $blkarray[$k];
- my $tail = substr $blkarray[$k], $len - $j * 2;
- if (exists $tails{$tail}) {
- push @{$tails{$tail}}, $k;
- } else {
- $tails{$tail} = [ $k ];
- }
- }
-
- # tails are calculated, now calculate the heads and merge.
- BLOCK:
- for $k (0 .. $#blkarray) {
- next if ! defined $blkarray[$k];
- my $tomerge = $k;
- while (1) {
- my $head = substr($blkarray[$tomerge], 0, $j * 2);
- my $entry = $tails{$head};
- next BLOCK if ! defined $entry;
-
- my $other = shift @{$entry};
- if ($other == $tomerge) {
- if (@{$entry}) {
- push @{$entry}, $other;
- $other = shift @{$entry};
- } else {
- push @{$entry}, $other;
- next BLOCK;
- }
- }
- if (@{$entry} == 0) {
- delete $tails{$head};
- }
-
- # a match was found
- my $merge = $blkarray[$other]
- . substr($blkarray[$tomerge], $j * 2);
- $blocklen -= $j;
- $blknum--;
-
- if ($other < $tomerge) {
- $blkarray[$tomerge] = undef;
- $blkarray[$other] = $merge;
- my $len = length $merge;
- my $tail = substr $merge, $len - $j * 2;
- $tails{$tail} = [ map { $_ == $tomerge ? $other : $_ }
- @{$tails{$tail}} ];
- next BLOCK;
- }
- $blkarray[$tomerge] = $merge;
- $blkarray[$other] = undef;
- }
- }
- }
- my $blockstr;
- for $k (0 .. $#blkarray) {
- $blockstr .= $blkarray[$k] if defined $blkarray[$k];
- }
-
- die "Unexpected $blocklen" if length($blockstr) != 2 * $blocklen;
- my $estimate = 2 * $blocklen + (0x20000 >> $i);
-
- printf " after merge %5d: %6d bytes\n", $blocklen, $estimate;
- if ($estimate < $bestest) {
- $bestest = $estimate;
- $bestshift = $i;
- $bestblkstr = $blockstr;
- }
-}
-
-my @blocks;
-my $blksize = 1 << $bestshift;
-for (my $j = 0; $j < 0x10000; $j += $blksize) {
- my $blkkey = substr $info, 2 * $j, 2 * $blksize;
- my $index = index $bestblkstr, $blkkey;
- while ($index & 1) {
- die "not found: $j" if $index == -1;
- $index = index $bestblkstr, $blkkey, $index + 1;
- }
- push @blocks, ($index / 2 - $j) & 0xffff;
-}
-
-# Phase 3: Generate the file
-die "UTF-8 limit of blocks may be exceeded: " . scalar(@blocks) . "\n"
- if @blocks > 0xffff / 3;
-die "UTF-8 limit of data may be exceeded: " . length($bestblkstr) . "\n"
- if length($bestblkstr) > 0xffff / 3;
-{
- print "Generating $ARGV[1] with shift of $bestshift";
- my ($i, $j);
-
- open OUTPUT, "> $ARGV[1]" or die "Failed creating output file: $!\n";
- print OUTPUT <<EOF;
-/* gnu/java/lang/CharData -- Database for java.lang.Character Unicode info
- Copyright (C) 2002 Free Software Foundation, Inc.
- *** This file is generated by scripts/unicode-muncher.pl ***
-
-This file is part of GNU Classpath.
-
-GNU Classpath is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2, or (at your option)
-any later version.
-
-GNU Classpath is distributed in the hope that it will be useful, but
-WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with GNU Classpath; see the file COPYING. If not, write to the
-Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-02111-1307 USA.
-
-Linking this library statically or dynamically with other modules is
-making a combined work based on this library. Thus, the terms and
-conditions of the GNU General Public License cover the whole
-combination.
-
-As a special exception, the copyright holders of this library give you
-permission to link this library with independent modules to produce an
-executable, regardless of the license terms of these independent
-modules, and to copy and distribute the resulting executable under
-terms of your choice, provided that you also meet, for each linked
-independent module, the terms and conditions of the license of that
-module. An independent module is a module which is not derived from
-or based on this library. If you modify this library, you may extend
-this exception to your version of the library, but you are not
-obligated to do so. If you do not wish to do so, delete this
-exception statement from your version. */
-
-package gnu.java.lang;
-
-/**
- * This contains the info about the unicode characters, that
- * java.lang.Character needs. It is generated automatically from
- * <code>$ARGV[0]</code>, by some
- * perl scripts. This Unicode definition file can be found on the
- * <a href="http://www.unicode.org">http://www.unicode.org</a> website.
- * JDK 1.4 uses Unicode version 3.0.0.
- *
- * The data is stored as string constants, but Character will convert these
- * Strings to their respective <code>char[]</code> components. The field
- * <code>BLOCKS</code> stores the offset of a block of 2<sup>SHIFT</sup>
- * characters within <code>DATA</code>. The DATA field, in turn, stores
- * information about each character in the low order bits, and an offset
- * into the attribute tables <code>UPPER</code>, <code>LOWER</code>,
- * <code>NUM_VALUE</code>, and <code>DIRECTION</code>. Notice that the
- * attribute tables are much smaller than 0xffff entries; as many characters
- * in Unicode share common attributes. Finally, there is a listing for
- * <code>TITLE</code> exceptions (most characters just have the same
- * title case as upper case).
- *
- * \@author scripts/unicode-muncher.pl (written by Jochen Hoenicke,
- * Eric Blake)
- * \@see Character
- */
-public interface CharData
-{
- /**
- * The Unicode definition file that was parsed to build this database.
- */
- String SOURCE = \"$ARGV[0]\";
-
- /**
- * The character shift amount to look up the block offset. In other words,
- * <code>(char) (BLOCKS.value[ch >> SHIFT] + ch)</code> is the index where
- * <code>ch</code> is described in <code>DATA</code>.
- */
- int SHIFT = $bestshift;
-
- /**
- * The mapping of character blocks to their location in <code>DATA</code>.
- * Each entry has been adjusted so that the 16-bit sum with the desired
- * character gives the actual index into <code>DATA</code>.
- */
- String BLOCKS
-EOF
-
- for ($i = 0; $i < @blocks / 11; $i++) {
- print OUTPUT $i ? "\n + \"" : " = \"";
- for $j (0 .. 10) {
- last if @blocks <= $i * 11 + $j;
- my $val = $blocks[$i * 11 + $j];
- print OUTPUT javaChar($val);
- }
- print OUTPUT "\"";
- }
-
- print OUTPUT <<EOF;
-;
-
- /**
- * Information about each character. The low order 5 bits form the
- * character type, the next bit is a flag for non-breaking spaces, and the
- * next bit is a flag for mirrored directionality. The high order 9 bits
- * form the offset into the attribute tables. Note that this limits the
- * number of unique character attributes to 512, which is not a problem
- * as of Unicode version 3.2.0, but may soon become one.
- */
- String DATA
-EOF
-
- my $len = length($bestblkstr) / 2;
- for ($i = 0; $i < $len / 11; $i++) {
- print OUTPUT $i ? "\n + \"" : " = \"";
- for $j (0 .. 10) {
- last if $len <= $i * 11 + $j;
- my $val = unpack "n", substr($bestblkstr, 2 * ($i*11 + $j), 2);
- print OUTPUT javaChar($val);
- }
- print OUTPUT "\"";
- }
-
- print OUTPUT <<EOF;
-;
-
- /**
- * This is the attribute table for computing the numeric value of a
- * character. The value is -1 if Unicode does not define a value, -2
- * if the value is not a positive integer, otherwise it is the value.
- * Note that this is a signed value, but stored as an unsigned char
- * since this is a String literal.
- */
- String NUM_VALUE
-EOF
-
- $len = @charinfo;
- for ($i = 0; $i < $len / 11; $i++) {
- print OUTPUT $i ? "\n + \"" : " = \"";
- for $j (0 .. 10) {
- last if $len <= $i * 11 + $j;
- my $val = $charinfo[$i * 11 + $j][0];
- print OUTPUT javaChar($val);
- }
- print OUTPUT "\"";
- }
-
- print OUTPUT <<EOF;
-;
-
- /**
- * This is the attribute table for computing the uppercase representation
- * of a character. The value is the signed difference between the
- * character and its uppercase version. Note that this is stored as an
- * unsigned char since this is a String literal.
- */
- String UPPER
-EOF
-
- $len = @charinfo;
- for ($i = 0; $i < $len / 11; $i++) {
- print OUTPUT $i ? "\n + \"" : " = \"";
- for $j (0 .. 10) {
- last if $len <= $i * 11 + $j;
- my $val = $charinfo[$i * 11 + $j][1];
- print OUTPUT javaChar($val);
- }
- print OUTPUT "\"";
- }
-
- print OUTPUT <<EOF;
-;
-
- /**
- * This is the attribute table for computing the lowercase representation
- * of a character. The value is the signed difference between the
- * character and its lowercase version. Note that this is stored as an
- * unsigned char since this is a String literal.
- */
- String LOWER
-EOF
-
- $len = @charinfo;
- for ($i = 0; $i < $len / 11; $i++) {
- print OUTPUT $i ? "\n + \"" : " = \"";
- for $j (0 .. 10) {
- last if $len <= $i * 11 + $j;
- my $val = $charinfo[$i * 11 + $j][2];
- print OUTPUT javaChar($val);
- }
- print OUTPUT "\"";
- }
-
- print OUTPUT <<EOF;
-;
-
- /**
- * This is the attribute table for computing the directionality class
- * of a character. At present, the value is in the range 0 - 18 if the
- * character has a direction, otherwise it is -1. Note that this is
- * stored as an unsigned char since this is a String literal.
- */
- String DIRECTION
-EOF
-
- $len = @charinfo;
- for ($i = 0; $i < $len / 11; $i++) {
- print OUTPUT $i ? "\n + \"" : " = \"";
- for $j (0 .. 10) {
- last if $len <= $i * 11 + $j;
- my $val = $charinfo[$i * 11 + $j][3];
- print OUTPUT javaChar($val);
- }
- print OUTPUT "\"";
- }
-
- print OUTPUT <<EOF;
-;
-
- /**
- * This is the listing of titlecase special cases (all other character
- * can use <code>UPPER</code> to determine their titlecase). The listing
- * is a sequence of character pairs; converting the first character of the
- * pair to titlecase produces the second character.
- */
- String TITLE
-EOF
-
- $len = length($titlecase) / 2;
- for ($i = 0; $i < $len / 11; $i++) {
- print OUTPUT $i ? "\n + \"" : " = \"";
- for $j (0 .. 10) {
- last if $len <= $i * 11 + $j;
- my $val = unpack "n", substr($titlecase, 2 * ($i*11 + $j), 2);
- print OUTPUT javaChar($val);
- }
- print OUTPUT "\"";
- }
-
- print OUTPUT ";\n}\n";
- close OUTPUT;
-}
-print "\nDone.\n";
projects / msp430-gcc.git / blobdiff