./ 0040775 0001130 0001130 00000000000 07550305777 007714 5 ustar krip krip ./Demo.jsp 0100644 0001130 0001130 00000011703 07445370772 011313 0 ustar krip krip <%@page import="org.openisis.*,java.io.IOException,java.util.Date"%>
<%!
static final String EMPTY = "";
static Db[] dbs = new Db[2];
static String dberr = EMPTY;
static {
try {
String[] args = new String[] { "-v", "error", "-encoding", "Cp850" };
dbs[0] = Db.open( "/var/db/cds/cds", args );
args = new String[] { "-v", "error", "-format", "aligned" };
dbs[1] = Db.open( "/var/db/unesb/unesb", args );
} catch (IOException e) {
dberr = e.toString();
}
};
static String NV ( String x ) { return null==x ? EMPTY : x; }
static int ROWS = 5; // number of rows to show at once
%>
<%
String search = NV( request.getParameter( "search" ) );
int off = 0;
int rows = ROWS;
int dbnum = -1;
try {
off = Integer.parseInt( request.getParameter( "off" ) );
} catch ( Exception e ) {}
try {
dbnum = Integer.parseInt( request.getParameter( "db" ) );
if ( dbs.length <= dbnum )
dbnum = -1;
} catch ( Exception e ) { }
if ( 0 > dbnum )
try {
dbnum = Integer.parseInt( (String)session.getAttribute( "db" ) );
} catch ( Exception e ) { dbnum = 0; }
else
session.setAttribute( "db", ""+dbnum );
Db db = dbs[dbnum];
try {
rows = Integer.parseInt( request.getParameter( "rows" ) );
} catch ( Exception e ) {}
int[] set = (int[])session.getAttribute( "set" );
%>
OpenIsis JSP demo
<%= dberr %>
Note: $,"",*,+,^,(F),(G),..,$$,/(tag) are supported.
Queries are processed straight left to right.
<%
if ( EMPTY != search ) {
long then = (new Date()).getTime();
int[] nset = db.search( search, Db.QRY_SIMPLE );
long now = (new Date()).getTime();
if ( null == nset ) {
%>
sorry, no results for <%=search%>
<%
session.removeAttribute( "set" );
return;
} else {
%>
got <%=nset.length%> rows for <%=search%>
in <%=(now - then)/1000.%> seconds.
<%
if ( null == set || null == request.getParameter("narrow") )
set = nset;
else { // and sets
set = Db.AND( set, nset );
%>
<%=set.length%> rows remaining after combine with previous search
<%
}
session.setAttribute( "set", set );
}
} // EMPTY != search
if ( null == set )
return;
if ( 0 < off ) {
int boff = off-rows;
if ( boff < 0 ) boff = 0;
%>
<<rows <%=boff+1%> to <%=boff+rows%>
<%
}
int end = off + rows;
if ( end >= set.length )
end = set.length;
else {
%>
rows <%=end+1%> to <%=end+rows%>>>
<%
}
%>
rows <%=off+1%> to <%=end%>
<%-- start output --%>
<%
if ( 0 == dbnum ) {
for ( int i=off; i
skip bad row <%=set[i]%> as <%=i+1%> of <%=set.length%>
<%
continue;
}
int id = r.getRowId();
int l = r.getLen();
%>
row <%=id%> with <%=l%> fields as <%=i+1%> of <%=set.length%>
<%
for ( int j=0; j
<%=id%>
<%=f.tag%>
<%=f.html()%>
<%
if ( '^' == f.val.charAt(0) ) {
Field[] s = f.split();
if ( null == s )
continue;
for ( int k=0; k
<%=id%>
<%=f.tag%>
<%=(char)s[k].tag%>
<%=s[k].html()%>
<%
} // for subfields
} // if subfields
} // for fields
} // for rows
%>
<% } %>
./Changes 0100644 0001130 0001130 00000000171 07355467174 011204 0 ustar krip krip Revision history for Perl extension OpenIsis.
0.01 Wed May 23 08:04:14 2001
- original version; created by h2xs 1.19
./Makefile 0100644 0001130 0001130 00000022235 07546641076 011353 0 ustar krip krip #
#/*
# openisis - an open implementation of the ISIS database
# Version 0.8.x (microversion see file Version)
# Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# This software is dedicated to the memory of Eckart Dietrich.
#
# This software is inspired by (but contains no code of) the iAPI
# Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
# See iAPI.txt for what it contains.
#EOH */
#
#
# $Id: Makefile,v 1.36 2002/10/02 19:00:14 kripke Exp $
# the Makefile
#
TARGETS=openisis
# object files in lib
LOBJS=lstr.o ldb.o lrec.o lfmt.o luti.o lio.o lqry.o
# classes in jar
CLASSES=Field Rec Db NativeDb PureDb Test Charset
# win32 dlls
DLLS=openjsis.dll
# standard make defines
CC=gcc
# you may want to try jikes
JC=javac
# figure out where to find JNI ...
ifeq ($(JNIINC),)
ifeq ($(JAVAHOME),)
JAVAHOME=/usr/java/jdk1.3
endif
JNIINC=-I$(JAVAHOME)/include -I$(JAVAHOME)/include/linux
endif
# check if some linker options for shared objects are given, you may need to
# specify some options on some platforms - e.g. "-G" on solaris.
ifeq ($(LDFLAGS),)
LDFLAGS=-shared
endif
# DBG=
ifeq ($(NDEBUG),)
DBG=-g
else
DBG=-DNDEBUG
endif
#UNIXIO=
UNIXIO=-DUNIXIO -D_REENTRANT
THREADDEF=
THREADLIB=
ifneq ($(PTHREADS),)
THREADDEF=-DHAVE_PTHREAD
THREADLIB=-lpthread
endif
# note that -O is necessary for -Wuninitialized warnings
# java needs long long
CFLAGS=-O2 -Wall -W -Wshadow -pedantic -Wno-long-long --unsigned-char \
$(DBG) $(JNIINC) $(UNIXIO) $(THREADDEF)
# DB to use -- define DB for db/DB/DB, or DBDIR for other path
ifeq ($(DB),)
DB=cds
endif
ifeq ($(DBDIR),)
DBDIR=db/$(DB)/$(DB)
endif
# standard default target
all: $(TARGETS)
.PHONY: all clean run demo perl jar java jdoc
# do not delete intermediates
.PRECIOUS: org/openisis/%.java %.class %.h
clean:
rm -rf *.o *.a *.so *.dll *.jar *.tmp *.exp org/openisis/*.class openisis demo core perl Makefile.old testout.txt
libopenisis.a: $(LOBJS)
$(AR) $(ARFLAGS) $@ $^
openisis: openisis.o libopenisis.a
$(CC) $(DBG) $(THREADLIB) -o $@ $^
shared: openisis.o libopenisis.so
$(CC) $(DBG) -o openisis $^
win32: openisis $(DLLS) openisis.jar
mv openisis openisis.exe
demo: demo.o libopenisis.a
$(CC) $(DBG) -o $@ $^
./demo -db $(DBDIR) 42
run: openisis
./openisis -v $${V:-warn} -db $(DBDIR) $(FMT) 2>&1 | less
dump: openisis
./openisis -v $${V:-warn} -db $(DBDIR) $(FMT)
recode: openisis
./openisis -v $${V:-warn} -db $(DBDIR) $(FMT) | recode 850/..Latin-1
check: openisis
./openisis -v $${V:-warn} -db $(DBDIR) -check
search: openisis
./openisis -v $${V:-warn} -search 'development$$' -db $(DBDIR) $(FMT) 2>&1 | less
searchdump: openisis
./openisis -v $${V:-warn} -search '$$' -db $(DBDIR) $(FMT) 2>&1
terms: openisis
./openisis -v $${V:-warn} -terms '$$' -db $(DBDIR) $(FMT) 2>&1 | less
test: openisis
./openisis -scan development -fmt prop -db $(DBDIR) >testout.txt
./openisis -terms '$$' -fmt prop -db $(DBDIR) >>testout.txt
./openisis -search 'devel$$' -fmt mfnf -db $(DBDIR) >>testout.txt
diff testout.txt testres.txt
crash: openisis
time ./openisis -v $${V:-warn} -crash 'Z$$' -db db/unesb/unesb
searchtest: openisis
./openisis -v $${V:-warn} -fmt mfnf -search 'a$$' -db $(DBDIR) $(FMT) >testout.txt
diff testout.txt searchtest.txt
pftest: openisis
./openisis -v $${V:-warn} -search 'Dacca' -db $(DBDIR) \
-pft "$${PFT:-`cat test.pft`}"
shsearch: shared
LD_LIBRARY_PATH=. ./openisis -v $${V:-debug} -search development -db $(DBDIR) $(FMT) 2>&1 | less
# take your time :)
time: openisis
free
time ./openisis -perf 100000 -db $(DBDIR) >/dev/null
free
# time results for the unesb perf test differ dramatically
# depending on the working of operating system caching
# you may achive about 0.40 sec (12.000 recs per sec)
# but immediatly after timing the cds,
# the same test may take more than 40 sec
utime: openisis
free
time ./openisis -perf 5000 -db db/unesb/unesb >/dev/null
free
time ./openisis -fmt mfnf -search 'k$$' -db db/unesb/unesb >/dev/null
#
perl: libopenisis.a
-mkdir perl
-ln -s ../Makefile.PL ../OpenIsis.pm ../OpenIsis.xs ../test.pl perl
cd perl; perl Makefile.PL; make test
:
: ++++++++++++++++++++++++++++++++++++++++++++++++++++++
: +++ as root in the perl subdir, run 'make install' +++
: ++++++++++++++++++++++++++++++++++++++++++++++++++++++
java: openisis.jar libopenjsis.so
:
jtest: java
LD_LIBRARY_PATH=. java -classpath openisis.jar:$(CLASSPATH) \
org.openisis.Test -v $${V:-warn} -db $(DBDIR) -encoding Cp850 \
-row 42 \
-search development \
-terms '' \
-mpu -row 130 \
-mhl -hta -row 130 \
-m1 -row 130 \
>testout.txt
diff testout.txt jtest.txt
jcrash: java
LD_LIBRARY_PATH=. time java -classpath openisis.jar:$(CLASSPATH) \
org.openisis.Test -v $${V:-warn} -dbpath db/unesb -db unesb \
-crash Z
# print in cds1.pft format
jcds1: java
LD_LIBRARY_PATH=. java -classpath openisis.jar:$(CLASSPATH) \
org.openisis.Test -v $${V:-warn} -db $(DBDIR) -encoding Cp850 \
-m1 -dump
jtime: java
LD_LIBRARY_PATH=. time java -classpath openisis.jar:$(CLASSPATH) \
org.openisis.Test -search 'a$$' -v $${V:-warn} \
-db db/unesb/unesb >/dev/null
jdump: java
LD_LIBRARY_PATH=. java -classpath openisis.jar:$(CLASSPATH) \
org.openisis.Test -v 9 -db $(DBDIR) 2>&1 | less
# -overview overview.html
jdoc:
- mkdir jdoc
javadoc \
-sourcepath . \
-encoding ISO-8859-1 \
-J-Xmx64m \
-d jdoc \
-use \
-version \
-author \
-windowtitle 'org.openisis source code documentation' \
-doctitle 'org.openisis source code documentation' \
-header 'org.openisis source code documentation' \
-footer 'this is public material' \
-bottom 'for your eyes only' \
-stylesheetfile jdocstyle.css \
-charset "iso-8859-1" \
org.openisis
charset: java
mkdir -p db/charset
LD_LIBRARY_PATH=. java -classpath openisis.jar:$(CLASSPATH) \
org.openisis.Charset -utf8seq >db/charset/charset.seq
mx seq=db/charset/charset.seq create=db/charset/charset -all now tell=1
mx db/charset/charset fst='1 1 MHL,V1' \
ln1=db/charset/charset.lk1 ln2=db/charset/charset.lk2 +fix -all now tell=1
ifload db/charset/charset db/charset/charset.lk1 db/charset/charset.lk2 \
+fix tell=1 -reset -balan
LD_LIBRARY_PATH=. java -classpath openisis.jar:$(CLASSPATH) \
org.openisis.Charset
# openisis.jar: $(foreach cls,$(CLASSES),org/openisis/$(cls).class)
openisis.jar: $(CLASSES:%=org/openisis/%.class)
jar cvf $@ org/openisis
%.class: %.java
$(JC) $<
%.h: %.class
javah -jni -o $@ $(subst /,.,$*)
libopenisis.so: $(LOBJS)
$(CC) $(LDFLAGS) -o $@ $^
libopenjsis.so: openjsis.o libopenisis.a
$(CC) $(LDFLAGS) -o $@ $^
%.dll:
$(CC) -mdll -o junk.tmp -Wl,--base-file,$*.tmp $($*_deps)
dlltool --dllname $@ --base-file $*.tmp \
--output-exp $*.exp --input-def $*.def
dlltool --dllname $@ --input-def $*.def \
--output-lib lib$*.a
gcc -mdll -mconsole -o $@ $($*_deps) -Wl,$*.exp,--enable-stdcall-fixup
# dependences of win32 dlls
# note on cross-builds: make sure you have the proper jni headers included -
# those supplied for the target platform
openjsis_deps=openjsis.o $(LOBJS)
openjsis.dll: $(openjsis_deps)
# maintenance targets ... most need a posix shell
ci: clean
: +++ enter checkin message +++
-read msg && ci -l -m"$$msg" *
tar: clean
@-[ -d tar ] || mkdir tar
tar czvf tar/openisis.`tail -1 Version`.tgz . --exclude CVS --exclude tar --exclude tcl --exclude db --exclude WEB-INF --exclude '.??*'
snap: clean
@-[ -d tar ] || mkdir tar
tar czvf tar/oi`date +%y%m%d`.tgz . --exclude CVS --exclude tar --exclude WEB-INF
headers:
for f in *.h *.c *.xs *.java; do \
cat Header.txt >$$f.tmp; \
sed -e '1,/^EOH/d' $$f >>$$f.tmp; \
mv -f $$f.tmp $$f; \
done
for f in Makefile Makefile.PL *.pl *.pm; do \
head -n 1 $$f >$$f.tmp; \
sed -e 's/^/#/' Header.txt >>$$f.tmp; \
sed -e '1,/^#EOH/d' $$f >>$$f.tmp; \
mv -f $$f.tmp $$f; \
done
# please do not try to make deps w/o sed, gcc and mv ...
deps:
sed -n -e '1,/^# generated deps/p' Makefile >Makefile.tmp
gcc -MM *.c >>Makefile.tmp
mv -f Makefile Makefile.old
mv Makefile.tmp Makefile
-diff Makefile Makefile.old
# generated deps -- do not edit below this line
demo.o: demo.c openisis.h
ioperf.o: ioperf.c
ldb.o: ldb.c lstr.h lio.h ldb.h luti.h openisis.h
lfmt.o: lfmt.c ldb.h luti.h openisis.h
lio.o: lio.c luti.h openisis.h lio.h
lqry.o: lqry.c ldb.h luti.h openisis.h
lrec.o: lrec.c ldb.h luti.h openisis.h
lstr.o: lstr.c lstr.h luti.h openisis.h
luti.o: luti.c lstr.h luti.h openisis.h
mlock.o: mlock.c
openisis.o: openisis.c openisis.h
openjsis.o: openjsis.c openisis.h org/openisis/NativeDb.h
tpl.o: tpl.c ldb.h luti.h openisis.h
./Header.txt 0100644 0001130 0001130 00000002126 07447676137 011647 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
./LGPL.txt 0100644 0001130 0001130 00000063476 07355467175 011232 0 ustar krip krip GNU LESSER GENERAL PUBLIC LICENSE
Version 2.1, February 1999
Copyright (C) 1991, 1999 Free Software Foundation, Inc.
59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
[This is the first released version of the Lesser GPL. It also counts
as the successor of the GNU Library Public License, version 2, hence
the version number 2.1.]
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
Licenses are intended to guarantee your freedom to share and change
free software--to make sure the software is free for all its users.
This license, the Lesser General Public License, applies to some
specially designated software packages--typically libraries--of the
Free Software Foundation and other authors who decide to use it. You
can use it too, but we suggest you first think carefully about whether
this license or the ordinary General Public License is the better
strategy to use in any particular case, based on the explanations below.
When we speak of free software, we are referring to freedom of use,
not price. Our General Public Licenses are designed to make sure that
you have the freedom to distribute copies of free software (and charge
for this service if you wish); that you receive source code or can get
it if you want it; that you can change the software and use pieces of
it in new free programs; and that you are informed that you can do
these things.
To protect your rights, we need to make restrictions that forbid
distributors to deny you these rights or to ask you to surrender these
rights. These restrictions translate to certain responsibilities for
you if you distribute copies of the library or if you modify it.
For example, if you distribute copies of the library, whether gratis
or for a fee, you must give the recipients all the rights that we gave
you. You must make sure that they, too, receive or can get the source
code. If you link other code with the library, you must provide
complete object files to the recipients, so that they can relink them
with the library after making changes to the library and recompiling
it. And you must show them these terms so they know their rights.
We protect your rights with a two-step method: (1) we copyright the
library, and (2) we offer you this license, which gives you legal
permission to copy, distribute and/or modify the library.
To protect each distributor, we want to make it very clear that
there is no warranty for the free library. Also, if the library is
modified by someone else and passed on, the recipients should know
that what they have is not the original version, so that the original
author's reputation will not be affected by problems that might be
introduced by others.
Finally, software patents pose a constant threat to the existence of
any free program. We wish to make sure that a company cannot
effectively restrict the users of a free program by obtaining a
restrictive license from a patent holder. Therefore, we insist that
any patent license obtained for a version of the library must be
consistent with the full freedom of use specified in this license.
Most GNU software, including some libraries, is covered by the
ordinary GNU General Public License. This license, the GNU Lesser
General Public License, applies to certain designated libraries, and
is quite different from the ordinary General Public License. We use
this license for certain libraries in order to permit linking those
libraries into non-free programs.
When a program is linked with a library, whether statically or using
a shared library, the combination of the two is legally speaking a
combined work, a derivative of the original library. The ordinary
General Public License therefore permits such linking only if the
entire combination fits its criteria of freedom. The Lesser General
Public License permits more lax criteria for linking other code with
the library.
We call this license the "Lesser" General Public License because it
does Less to protect the user's freedom than the ordinary General
Public License. It also provides other free software developers Less
of an advantage over competing non-free programs. These disadvantages
are the reason we use the ordinary General Public License for many
libraries. However, the Lesser license provides advantages in certain
special circumstances.
For example, on rare occasions, there may be a special need to
encourage the widest possible use of a certain library, so that it becomes
a de-facto standard. To achieve this, non-free programs must be
allowed to use the library. A more frequent case is that a free
library does the same job as widely used non-free libraries. In this
case, there is little to gain by limiting the free library to free
software only, so we use the Lesser General Public License.
In other cases, permission to use a particular library in non-free
programs enables a greater number of people to use a large body of
free software. For example, permission to use the GNU C Library in
non-free programs enables many more people to use the whole GNU
operating system, as well as its variant, the GNU/Linux operating
system.
Although the Lesser General Public License is Less protective of the
users' freedom, it does ensure that the user of a program that is
linked with the Library has the freedom and the wherewithal to run
that program using a modified version of the Library.
The precise terms and conditions for copying, distribution and
modification follow. Pay close attention to the difference between a
"work based on the library" and a "work that uses the library". The
former contains code derived from the library, whereas the latter must
be combined with the library in order to run.
GNU LESSER GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License Agreement applies to any software library or other
program which contains a notice placed by the copyright holder or
other authorized party saying it may be distributed under the terms of
this Lesser General Public License (also called "this License").
Each licensee is addressed as "you".
A "library" means a collection of software functions and/or data
prepared so as to be conveniently linked with application programs
(which use some of those functions and data) to form executables.
The "Library", below, refers to any such software library or work
which has been distributed under these terms. A "work based on the
Library" means either the Library or any derivative work under
copyright law: that is to say, a work containing the Library or a
portion of it, either verbatim or with modifications and/or translated
straightforwardly into another language. (Hereinafter, translation is
included without limitation in the term "modification".)
"Source code" for a work means the preferred form of the work for
making modifications to it. For a library, complete source code means
all the source code for all modules it contains, plus any associated
interface definition files, plus the scripts used to control compilation
and installation of the library.
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running a program using the Library is not restricted, and output from
such a program is covered only if its contents constitute a work based
on the Library (independent of the use of the Library in a tool for
writing it). Whether that is true depends on what the Library does
and what the program that uses the Library does.
1. You may copy and distribute verbatim copies of the Library's
complete source code as you receive it, in any medium, provided that
you conspicuously and appropriately publish on each copy an
appropriate copyright notice and disclaimer of warranty; keep intact
all the notices that refer to this License and to the absence of any
warranty; and distribute a copy of this License along with the
Library.
You may charge a fee for the physical act of transferring a copy,
and you may at your option offer warranty protection in exchange for a
fee.
2. You may modify your copy or copies of the Library or any portion
of it, thus forming a work based on the Library, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) The modified work must itself be a software library.
b) You must cause the files modified to carry prominent notices
stating that you changed the files and the date of any change.
c) You must cause the whole of the work to be licensed at no
charge to all third parties under the terms of this License.
d) If a facility in the modified Library refers to a function or a
table of data to be supplied by an application program that uses
the facility, other than as an argument passed when the facility
is invoked, then you must make a good faith effort to ensure that,
in the event an application does not supply such function or
table, the facility still operates, and performs whatever part of
its purpose remains meaningful.
(For example, a function in a library to compute square roots has
a purpose that is entirely well-defined independent of the
application. Therefore, Subsection 2d requires that any
application-supplied function or table used by this function must
be optional: if the application does not supply it, the square
root function must still compute square roots.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Library,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Library, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote
it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Library.
In addition, mere aggregation of another work not based on the Library
with the Library (or with a work based on the Library) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may opt to apply the terms of the ordinary GNU General Public
License instead of this License to a given copy of the Library. To do
this, you must alter all the notices that refer to this License, so
that they refer to the ordinary GNU General Public License, version 2,
instead of to this License. (If a newer version than version 2 of the
ordinary GNU General Public License has appeared, then you can specify
that version instead if you wish.) Do not make any other change in
these notices.
Once this change is made in a given copy, it is irreversible for
that copy, so the ordinary GNU General Public License applies to all
subsequent copies and derivative works made from that copy.
This option is useful when you wish to copy part of the code of
the Library into a program that is not a library.
4. You may copy and distribute the Library (or a portion or
derivative of it, under Section 2) in object code or executable form
under the terms of Sections 1 and 2 above provided that you accompany
it with the complete corresponding machine-readable source code, which
must be distributed under the terms of Sections 1 and 2 above on a
medium customarily used for software interchange.
If distribution of object code is made by offering access to copy
from a designated place, then offering equivalent access to copy the
source code from the same place satisfies the requirement to
distribute the source code, even though third parties are not
compelled to copy the source along with the object code.
5. A program that contains no derivative of any portion of the
Library, but is designed to work with the Library by being compiled or
linked with it, is called a "work that uses the Library". Such a
work, in isolation, is not a derivative work of the Library, and
therefore falls outside the scope of this License.
However, linking a "work that uses the Library" with the Library
creates an executable that is a derivative of the Library (because it
contains portions of the Library), rather than a "work that uses the
library". The executable is therefore covered by this License.
Section 6 states terms for distribution of such executables.
When a "work that uses the Library" uses material from a header file
that is part of the Library, the object code for the work may be a
derivative work of the Library even though the source code is not.
Whether this is true is especially significant if the work can be
linked without the Library, or if the work is itself a library. The
threshold for this to be true is not precisely defined by law.
If such an object file uses only numerical parameters, data
structure layouts and accessors, and small macros and small inline
functions (ten lines or less in length), then the use of the object
file is unrestricted, regardless of whether it is legally a derivative
work. (Executables containing this object code plus portions of the
Library will still fall under Section 6.)
Otherwise, if the work is a derivative of the Library, you may
distribute the object code for the work under the terms of Section 6.
Any executables containing that work also fall under Section 6,
whether or not they are linked directly with the Library itself.
6. As an exception to the Sections above, you may also combine or
link a "work that uses the Library" with the Library to produce a
work containing portions of the Library, and distribute that work
under terms of your choice, provided that the terms permit
modification of the work for the customer's own use and reverse
engineering for debugging such modifications.
You must give prominent notice with each copy of the work that the
Library is used in it and that the Library and its use are covered by
this License. You must supply a copy of this License. If the work
during execution displays copyright notices, you must include the
copyright notice for the Library among them, as well as a reference
directing the user to the copy of this License. Also, you must do one
of these things:
a) Accompany the work with the complete corresponding
machine-readable source code for the Library including whatever
changes were used in the work (which must be distributed under
Sections 1 and 2 above); and, if the work is an executable linked
with the Library, with the complete machine-readable "work that
uses the Library", as object code and/or source code, so that the
user can modify the Library and then relink to produce a modified
executable containing the modified Library. (It is understood
that the user who changes the contents of definitions files in the
Library will not necessarily be able to recompile the application
to use the modified definitions.)
b) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (1) uses at run time a
copy of the library already present on the user's computer system,
rather than copying library functions into the executable, and (2)
will operate properly with a modified version of the library, if
the user installs one, as long as the modified version is
interface-compatible with the version that the work was made with.
c) Accompany the work with a written offer, valid for at
least three years, to give the same user the materials
specified in Subsection 6a, above, for a charge no more
than the cost of performing this distribution.
d) If distribution of the work is made by offering access to copy
from a designated place, offer equivalent access to copy the above
specified materials from the same place.
e) Verify that the user has already received a copy of these
materials or that you have already sent this user a copy.
For an executable, the required form of the "work that uses the
Library" must include any data and utility programs needed for
reproducing the executable from it. However, as a special exception,
the materials to be distributed need not include anything that is
normally distributed (in either source or binary form) with the major
components (compiler, kernel, and so on) of the operating system on
which the executable runs, unless that component itself accompanies
the executable.
It may happen that this requirement contradicts the license
restrictions of other proprietary libraries that do not normally
accompany the operating system. Such a contradiction means you cannot
use both them and the Library together in an executable that you
distribute.
7. You may place library facilities that are a work based on the
Library side-by-side in a single library together with other library
facilities not covered by this License, and distribute such a combined
library, provided that the separate distribution of the work based on
the Library and of the other library facilities is otherwise
permitted, and provided that you do these two things:
a) Accompany the combined library with a copy of the same work
based on the Library, uncombined with any other library
facilities. This must be distributed under the terms of the
Sections above.
b) Give prominent notice with the combined library of the fact
that part of it is a work based on the Library, and explaining
where to find the accompanying uncombined form of the same work.
8. You may not copy, modify, sublicense, link with, or distribute
the Library except as expressly provided under this License. Any
attempt otherwise to copy, modify, sublicense, link with, or
distribute the Library is void, and will automatically terminate your
rights under this License. However, parties who have received copies,
or rights, from you under this License will not have their licenses
terminated so long as such parties remain in full compliance.
9. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Library or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Library (or any work based on the
Library), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Library or works based on it.
10. Each time you redistribute the Library (or any work based on the
Library), the recipient automatically receives a license from the
original licensor to copy, distribute, link with or modify the Library
subject to these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties with
this License.
11. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Library at all. For example, if a patent
license would not permit royalty-free redistribution of the Library by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Library.
If any portion of this section is held invalid or unenforceable under any
particular circumstance, the balance of the section is intended to apply,
and the section as a whole is intended to apply in other circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
12. If the distribution and/or use of the Library is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Library under this License may add
an explicit geographical distribution limitation excluding those countries,
so that distribution is permitted only in or among countries not thus
excluded. In such case, this License incorporates the limitation as if
written in the body of this License.
13. The Free Software Foundation may publish revised and/or new
versions of the Lesser General Public License from time to time.
Such new versions will be similar in spirit to the present version,
but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Library
specifies a version number of this License which applies to it and
"any later version", you have the option of following the terms and
conditions either of that version or of any later version published by
the Free Software Foundation. If the Library does not specify a
license version number, you may choose any version ever published by
the Free Software Foundation.
14. If you wish to incorporate parts of the Library into other free
programs whose distribution conditions are incompatible with these,
write to the author to ask for permission. For software which is
copyrighted by the Free Software Foundation, write to the Free
Software Foundation; we sometimes make exceptions for this. Our
decision will be guided by the two goals of preserving the free status
of all derivatives of our free software and of promoting the sharing
and reuse of software generally.
NO WARRANTY
15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Libraries
If you develop a new library, and you want it to be of the greatest
possible use to the public, we recommend making it free software that
everyone can redistribute and change. You can do so by permitting
redistribution under these terms (or, alternatively, under the terms of the
ordinary General Public License).
To apply these terms, attach the following notices to the library. It is
safest to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least the
"copyright" line and a pointer to where the full notice is found.
Copyright (C)
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
Also add information on how to contact you by electronic and paper mail.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the library, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the
library `Frob' (a library for tweaking knobs) written by James Random Hacker.
, 1 April 1990
Ty Coon, President of Vice
That's all there is to it!
./MANIFEST 0100644 0001130 0001130 00000000075 07355467174 011045 0 ustar krip krip Changes
Makefile.PL
MANIFEST
OpenIsis.pm
OpenIsis.xs
test.pl
./Makefile.PL 0100644 0001130 0001130 00000003232 07447676137 011667 0 ustar krip krip #
#/*
# openisis - an open implementation of the ISIS database
# Version 0.8.x (microversion see file Version)
# Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# This software is dedicated to the memory of Eckart Dietrich.
#
# This software is inspired by (but contains no code of) the iAPI
# Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
# See iAPI.txt for what it contains.
#EOH */
#
# $Id: Makefile.PL,v 1.8 2002/03/25 19:36:44 kripke Exp $
# the MakeMaker Makefile for the openisis perl XSUB
#
use ExtUtils::MakeMaker;
# See lib/ExtUtils/MakeMaker.pm for details of how to influence
# the contents of the Makefile that is written.
WriteMakefile(
'NAME' => 'OpenIsis',
'VERSION_FROM' => 'OpenIsis.pm', # finds $VERSION
'LIBS' => ['-L.. -lopenisis'], # e.g., '-lm'
'DEFINE' => '', # e.g., '-DHAVE_SOMETHING'
'INC' => '-I..', # e.g., '-I/usr/include/other'
'CC' => 'gcc',
'LD' => 'gcc'
);
./OpenIsis.pm 0100644 0001130 0001130 00000006431 07505604210 011762 0 ustar krip krip #
#/*
# openisis - an open implementation of the ISIS database
# Version 0.8.x (microversion see file Version)
# Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# This software is dedicated to the memory of Eckart Dietrich.
#
# This software is inspired by (but contains no code of) the iAPI
# Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
# See iAPI.txt for what it contains.
#EOH */
#
# $Id: OpenIsis.pm,v 1.10 2002/06/24 11:22:26 kripke Exp $
# perl module wrapper for openisis perl XSUB
#
package OpenIsis;
use strict;
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK);
require Exporter;
require DynaLoader;
require AutoLoader;
@ISA = qw(Exporter DynaLoader);
# Items to export into callers namespace by default. Note: do not export
# names by default without a very good reason. Use EXPORT_OK instead.
# Do not simply export all your public functions/methods/constants.
@EXPORT = qw(
);
$VERSION = '0.852';
bootstrap OpenIsis $VERSION;
# query mode constants
*QRY_KEYEQ = \0;
*QRY_KEYPF = \1;
*QRY_KEYAT = \2;
*QRY_SCANE = \64;
*QRY_SCANC = \65;
*QRY_SIMPLE = \128;
*QRY_PROPER = \129;
# Preloaded methods go here.
# Autoload methods go after =cut, and are processed by the autosplit program.
1;
__END__
# Below is the stub of documentation for the OpenIsis module.
=head1 NAME
OpenIsis - Perl extension for accessing ISIS databases
=head1 SYNOPSIS
use OpenIsis;
$db = OpenIsis::open( 'basename', '-dbpath', '/home/me/mydb' );
$maxmfn = OpenIsis::maxRowid( $db );
for (OpenIsis::query( $db, 'plant * water' )) {
my $row = OpenIsis::read( $db, $_ ); # get hashref
$row = OpenIsis::read( $db, $_, 'MHL V24,V26,V71' ); # read formatted
my $v24 = $row->{'24'}; # field tags map to arrayrefs
OpenIsis::MHL( @$v24 ); # apply heading mode to array
print $row->{'mfn'}, "TITLE :", join("\n ", @$v24 ), "\n";
my $v30 = $row->{'30'}->[0]; # contents is like ^ap. 233-238 ^billus.
my $subfields = OpenIsis::subfields( $v30 ); # get subfield values
print "a = ", $subfields->{'a'}, " b = ", $subfields->{'b'}, "\n";
}
# print terms starting with A
print join('; ',OpenIsis::terms( $db, 'a$' )), "\n";
=head1 DESCRIPTION
The OpenIsis perl extension provides access to ISIS databases
based on the openisis C library.
=head1 AUTHOR
Erik Grziwotz, erik@openisis.org
=head1 SEE ALSO
perl(1).
=cut
sub MHL {
for ( @_ ) {
s/(<[^=>]*)=[^>]+>/\1>/g; # dump substitutions
s/>< pairs
s/[><]//g; # nuke other ><
s/^\^.//; # kill initial subfield spec
s/\^a/; /; # ^a -> ;
s/\^[b-i]/, /g; # ^[b-i] -> ,
s/\^./. /g; # others -> .
}
}
./OpenIsis.xs 0100644 0001130 0001130 00000010676 07505604210 012006 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
/*
$Id: OpenIsis.xs,v 1.11 2002/06/24 11:22:26 kripke Exp $
the openisis XSUB
*/
#include "EXTERN.h"
#include "perl.h"
#include "XSUB.h"
#include /* perl's assert doesn't really assert */
#include "openisis.h"
MODULE = OpenIsis PACKAGE = OpenIsis
PROTOTYPES: ENABLE
void
log( level, file )
int level
char* file
CODE:
openIsisLog( level, *file ? file : NULL );
int
open( name, ... )
char* name
CODE:
{
const char **argv = NULL;
int argc = 0;
if ( 1 < items ) {
STRLEN n_a;
int i;
argv = (const char **)malloc( (items-1) * sizeof(argv[0]) );
for ( i=1; irowid ), 0 );
if ( items > 2 ) {
char *fmt = (char *)SvPV_nolen( ST(2) );
OpenIsisRec *q;
OPENISIS_INITBUF(x);
q = openIsisFmt( &x.r, fmt, r );
free( r );
r = q;
}
for ( i=0; i < r->len; i++ ) {
char buf[7];
AV *ar;
SV *ref, **entry;
SV *val;
val = newSVpv( (char*)r->field[i].val, r->field[i].len );
snprintf( buf, sizeof(buf), "%hu", r->field[i].tag );
buf[sizeof(buf)-1] = 0;
entry = hv_fetch( hv, buf, strlen(buf), 1 );
assert ( NULL != entry ); /* out of memory */
if ( NULL == *entry || ! SvROK(*entry) ) {
ar = newAV();
*entry = newRV_noinc((SV*)ar);
} else {
ar = (AV*)SvRV(*entry);
assert( SVt_PVAV == SvTYPE(ar) );
}
av_push( ar, val );
}
if ( r != &x.r )
free( r );
}
RETVAL = newRV_noinc( (SV*)hv );
}
OUTPUT:
RETVAL
SV*
subfields( field )
char *field
CODE:
{
OpenIsisField f;
OpenIsisRec *r;
HV *hv = newHV();
f.tag = 0;
f.val = field;
f.len = strlen(field);
r = openIsisReadField( NULL, &f );
if ( NULL != r ) {
int i;
char buf[2];
buf[sizeof(buf)-1] = 0;
for ( i=0; i < r->len; i++ ) {
buf[0] = (char)r->field[i].tag;
hv_store( hv, buf, buf[0] ? 1 : 0,
newSVpv( (char*)r->field[i].val, r->field[i].len ), 0 );
}
free( r );
}
RETVAL = newRV_noinc( (SV*)hv );
}
OUTPUT:
RETVAL
void
query( db, key, ... )
int db
char *key
PPCODE:
{
long mode = OPENISIS_QRY_SIMPLE;
long skip = 0;
int got;
OpenIsisSet set;
set.len = 0;
got = (int)openIsisQuery( &set, db, key, mode, skip );
if ( 0 < got ) {
int i;
EXTEND(SP,got);
for ( i=0; iOpenIsis 0.8.xcurrent versionisisopenisishowtoaboutlinksen espanol
win32 (0.8.6)
( mingw32 cross-build) should work on all win32 platforms from win95b on.
Note: If you want to use java you will need the JDK1.3.x
The current windows version is not yet thread-safe, so using it with a Java
servlet engine under windows requires some care.
solaris (0.8.4)
( build on solaris 5.8 )
Please mail us
if you need a more up-to-date solaris binary.
Note: the JDK shipped with solaris 5.7 / 5.8 ( java 1.2 ) should work.
News:
0.8.7 October 2002
Version 0.8.7 supports writing of DOS/WinIsis masterfiles and xrf.
This currently works fine for a single process on Linux.
See writing.txt for details.
Current TODOs:
interlock multiprocess writing (PHP, Perl CGI)
and fix the windows and solaris versions.
Besides writing support, there is a new streaming record reader,
which groks various formats like the SYSPAR.PAR, email headers and
property files, so you can fill your db from such textfiles.
Next step: new indexing engine.
August 2002
No new software yet. Still very busy doing metawork.
We are preparing to set up an organisation to support OpenIsis
development with much more momentum and a company for professional
services like help on large scale ISIS installations.
0.8.6 June 2002
This version supports basic formatting. While most, especially graphical,
features of WinISIS or CISIS formatting are not yet supported
(which are typically not used in a web environment anyway),
there is support for repeated subfields as declared by MARC for many fields.
See formatting notes for details.
The perl binding supports formatting (see the test.pl),
enhanced versions for all those languages are to follow soon.
PHP June 2002 Braulio Jos Solano Rojas
from Costa Rica created a PHP binding, which can be seen in action at
galileo.
Available as download
or from sourceforge module php-openisis.
Also, the Institut Teknologi Bandung of Indonesia
switched it's web index to a PHP/OpenIsis based version.
0.8.5.2 March 2002
Much enhanced PERL binding. See the OpenIsis.pm included in the sources.
0.8.5.1 March 2002
Java now has support for basic formatting modes like MHL,
various HTML-safety modes (like escaping all non-ASCIIs),
a Vn-field-selector-style method
and several nice utils.
Indentation is not properly handled, since there is no easy
common solution in HTML. Will build tools for a choice of
standard strategies later ...
0.8.5 March 2002
Finally some implementation of the query language
(demo).
All the operators are there (including /(tag), but not /(t1,t2...)).
Every attempt is made to limit the potential costs of even extremely
stupid queries like "$"^"$", so no historical (#n) or intermediate
results (for precedence) are stored.
Queries are processed strictly left to right on a buffer of 1000 hits.
0.8.4 January 2002
Nearly complete rewrite of search code with support for NEAR conditions.
Fixed alignment problems in IFP, now works with the unesb db
(-format aligned) and the cds db as distributed with winisis.
The cds db we had with previous versions (from an old CDS distribution)
has a mixed format: aligned leaf files, others unaligned.
Although support for a mixed format is easily achieved with openisis,
it will not be included unless somebody has a need for it (send us mail).
The JSP demo now supports
searching the unesb db
with over 70.000 rows (note the hosting server is a 500MHz Celeron).
Searching is limited to 1000 postings, usually resulting in a
somewhat lower number of rows (where rows have multiple matching postings).
The lowest row number (MFN) that was cut off is recorded,
and it is possible (not yet in the JSP) to repeat the search
starting from that rowid.
0.8.3 October 2001
First truly usable release, since we have true search by index now
(as prefix or complete word).
Search gives a list (array) of sorted MFNs;
arithmetic on those lists (and, or, not) is straightforward.
The JSP demo
shows how a query is refined (narrowed) iteratively
by ANDing it with a second query.
And thanks to Veronica Lencinas and colleagues,
we have a version of this document en espanol.
September 2001
Not a new release yet, but maintenance and testing.
New structure logging mechanism.
Sources are available via CVS at
sourceforge.
Windows version openisis.exe running.
0.8.2 August 2001
openisis now under LGPL, no legacy code.
Conversion from file structures governed by abstract dynamic description
rather than C-structs, so we can support different file layouts,
much larger databases, big endian processors and more.
Simple full-scan search available in C-Lib and Java.
Given a random read throughput of about 30.000 Records/sec on a
lame 300 MHz Notebook this seems to be of practic use.
jsp demo under development ;).
0.8.1 June 2001
Java native interface version available.
Java package org.openisis has Db, Rec, Field, Test.
NativeDb implemented in libopenjsis.so.
Subfield splitting and htmlifying in pure java.
0.8.0 May 2001
Subfield splitting and htmlifying.
Everything also available from perl as an xsub.
Record shows up as hash, handy but no repeated fields.
Makefile.PL, test.pl etc.
0.7.9 May 2001
First version:
static C-Library libopenisis.a for reading records by rowid(Mfn).
Executable "openisis" does test.
Logging, argumentparsing, Makefile, demo etc.
Isis is a simple, yet powerful database system with a large installed
base since the 80s. Since it's well suited for bibliographic data,
it's commonly used in libraries, and since it's very low cost,
especially in those running on a low budget.
An isis DB is a list of rows of unspecified structure,
each identified by a unique number, the rowid (a.k.a. mfn).
Each row is a list of fields, and each field has number (tag)
and a string value. Within a row there may be zero, one or more
fields with a given tag. While the field's value usually is a
textual representation of data in one or the other character
encoding (commonly one of the IBM/DOS code pages), it may
actually contain arbitrary bytes.
subfields
There is a convention to encode multiple fields in one by separating
them with a '^' followed by one character tagging the subfield.
So the field value '^afoo^bbar^bbaz' represents a field having
one 'a' subfield with value 'foo' and two 'b' subfields 'bar' and 'baz'.
(An other separator char may be used, e.g. '$' is used in the MARC standard).
formatting
There is a formatting language, with literal text, field and subfield
variables, if-else branches (on field existance)
and for loops (over field repetitions) (roughly speaking).
indexing
An index is build by converting a row into a list of words
(optionally applying formats) and stuffing every word,
qualified by the position of it's occurence in the row,
into a B+-Tree (which is actually spread to six files).
Searching for a word or
word prefix is possible with or without qualifying the position (field).
Since all fields can be combined into one index, it is usually not
necessary (but possible) to set up multiple indexes.
queries
A query language allows for combination of word lookups using
and, or and not(without)
operators.
usage
While isis lacks most features of RDBMS like complex relations
between different entities, it's flexibility comes in handy for
many catalogues and directories with highly varying records and
one single level of substructure, which today are usually
modelled in XML documents rather than table rows.
In other words, isis is an ideal storage for many XML applications.
The flexible indexing mechanism combines the best of full text
searching and structured retrieval.
The mother of all isis software is a DOS version of "MicroISIS"
as an integrated system with textual user interface.
There is a BSD version of "CDS/ISIS" which
also runs under linux up to some 2.2.x kernels
(current 2.4 kernels do not support the iBCS module for COFF binaries).
Then there are several versions of "WinISIS" (M$-Windows only,
but runs under linux/wine).
A shared library version "isis.dll" of functions
to access an isis db from your code exists, despite it's name,
also in a linux version ("isilux");
however, you need some pretty special libs to make it run.
A set of command line tools ("cisis") performs tasks like importing
ISO2709 bibliographic databases, inverting (index building) etc.
The thing next to an isis database server is "wwwisis",
which runs as CGI or from the command line and performs
most isis tasks (win and lin versions). However, it actually
runs per request, not as a server itself, and thus cannot
provide concurrency control.
This "official" isis software, which is maintained by
Unesco and/or
Bireme,
is accompanied by a couple of independent developments,
some of which are in the public domain.
Javaisis
is an AWT-based GUI (3.5 uses SWING) and a corresponding server,
which in turn uses wwwisis.
Robert Janusz
wrote a C-lib (iAPI) from scratch,
which was the starting point for the openisis software.
So why are we writing the openisis software?
Because Isis is not open source software, it's not even free software,
and that leads to a whole bunch of problems.
Availability (in theory)
Versions of the software exist for some operating systems,
library versions and languages.
For other environments, there is no version of the software,
and there is not much one can do about it.
Availability (in practice)
You may download most the software, but it's partly protected with passwords,
which you have to order at some national distributor.
You have to pay some fee and/or declare some good reasons,
why you want to use the software.
Then you have to wait. In germany, for example,
it didn't work at all for some time, until the newly founded
Isisnetz remedied the situation.
Availability (in legal terms)
Some parts of the software are accompanied by different documents
stating some license terms, others are not.
Terms seem to be pretty different between countries.
One can not easily figure out, what exactly might be allowed usage.
Availability (of documentation)
Some documentation is available in english,
some only in portugese, espanol or italiano.
Only a small part is downloadable at all, most is paperware.
Bugfixing
There is no way one can fix a bug,
and not much one can do about having somebody fix it.
Extending
The only way one could write a Binding for perl or Java
would be using the isis.dll.
There are problems with regard to required additional libraries
(especially some C++ stuff), there are no statements about
thread safety, unicode compatibility and so on.
As a consequence, it's practically impossible to write a
state-of-the-art web application based on an isis db.
Improving
Many users develop useful ideas for improvements from practice.
Their expertise is lost as they are not able to turn it into
improved software.
Enabling
While open source software enables people all over the world
to shape their tools themselves, closed software lets them in
dependency.
To address these problems we feel the need for an open source
implementation of isis.
Of course it would be best to have all of the existing isis code
under one or the other form of open license (GPL, LGPL, artistic
or similar as appropriate).
On the other hand, an independent secondary implementation has
advantages in it's own right. It may have a different focus
and develop strengths in one aspect while another aproach
performs better in other situations. For example,
openisis will have some support for multithreading and unicode,
which is paid for by a certain overhead.
A rewrite by developers with a different background might
introduce new ideas which finally, after having had their
indepent test bed, help improve the standard.
OpenIsis as a software to access isis databases is and will be freely
available for everybody with full sourcecode, no fee, no restrictions.
In general, there are no plans to reimplement every piece
of code ever written for isis. To be of practical value,
OpenIsis has to maintain compatibility in the format of
the database files anyway. So, one may use winisis or
whatever existing import scripts to create and maintain
the database, yet deploy OpenIsis' perl interface to run
powerful reports and the Java Native Interface to allow
queries from a Servlet based web application.
OpenIsis will focus on providing tools rather than applications.
For example, there will be no attempt to mirror functionality
of winisis unless the GUI toolkit is done. To achive this,
OpenIsis provides access from the most important programming
languages: Java for the web (DONE), perl for the scripts (DONE) and
Tcl/Tk for platform independent GUIs (partly DONE).
All others can, of course, link the lib.
Next steps:
make file layout configurable to allow for larger db (DONE)
implement searching (full-scan searching DONE)
implement index-based searching (DONE)
more performance: try std (DONE, performs badly)
and homegrown io buffering,
further accelerate loop in ldb's convert function
start work on thread-save pure-java implementation (volunteers ?)
prepare binary releases for windows (.exe and .dll for java) (DONE)
Start by downloading the Software.
Unpack everything in some arbitrary directory.
For the tests, you will also need some isis database,
which must be located as files db/cds/cds.*.
Try this one.
Make sure filenames are lowercase.
If you are on Windows, you should either get yourself the cygwin
environment with tools like gmake and gcc or volunteer as a porter
and start writing the Makefile for your make and compiler.
Erik has build a Windows version using mingw and Linux gcc as crosscompiler.
If you are on Linux, everything is fine. On a MAC ... no idea.
Type "make" and enjoy the compiler messages.
(If your make complains, e.g. on BSD, try "gmake").
Type "make demo" and enjoy your first open isis record.
(You installed a db/cds/cds.*, didn't you? It has 42 rows?)
Type "make run" and watch the guts of your db passing by.
Type "make test", there should be no difference between the testout.txt
and the testres.txt as provided
(using this cds database from winisis
and this 15 MB 70.000+ row unesb.zip db).
Type "make time" to measure performance,
subsequent tries are usually much faster.
My 800 MHz P3 random-reads more than 80.000 records a second,
once the files are in the system cache.
Typical values:
time ./openisis -perf 100000 -db db/cds/cds >/dev/null
real 0m1.851s
user 0m0.930s
sys 0m0.920s
time ./openisis -perf 5000 -db db/unesb/unesb -format aligned >/dev/null
real 0m1.623s
user 0m0.140s
sys 0m0.250s
time ./openisis -fmt mfnf -search 'k$' -db db/unesb/unesb -format aligned >/dev/null
860 rows for k
real 0m0.223s
user 0m0.050s
sys 0m0.070s
Type "make perl" to build the perl stuff;
some perl 5.* must be installed beforehand.
Type "make java" or, if you just can't get enough, "make jdump",
to see it all happen in your shiny new JDK1.3 Java VM.
Some 1.2.* JDKs should do, but tell the Makefile to not
look in /usr/java/jdk1.3 by setting JAVAHOME.
libopenisis.a can be linked with your code; no installation necessary.
You may wan't to install the 'openisis' binary somewhere in your path
for the fun of it; go ahead, just copy, no magic registry entries.
To install the perl stuff for general availability in your
/usr/lib/perl5 or whatever, cd to the perl subdir (after "make perl")
and issue "make install" (as root or otherwise legitimated).
After that, try "perldoc OpenIsis" and the demo.pl script.
Java, like perl, needs to dynamically slurp both some stuff in
the own language and a native shared object.
The former is openisis.jar, set your CLASSPATH to include it,
or specify when invoking java like in the Makefile.
The latter is libopenjsis.so on linux (yes, it's jsis).
The system dynamic linker must be able to find it;
see NativeDb.java for details.
OpenIsis.org is sponsored by
,
a service of merconic, Berlin, Germany.
As a student's site, allmaxx supports open software with a focus on
education and knowledge management.
See also the open community for science.
Currently the site is maintained by
Erik and Paul.
Volunteers are very welcome.
Openisis sources are available at
side by side with Franck Martin's PHP isis project. Thanks, Franck!
ISIS, charsets and unicode
$Revision: 1.28 $ last changed $Date: 2002/10/07 13:28:28 $ by $Author: kripke $
(this page intentionally left blank :)
./Version 0100644 0001130 0001130 00000000041 07550303061 011232 0 ustar krip krip #define OPENISIS_VERSION \
0.8.7
./demo.c 0100644 0001130 0001130 00000003441 07447676137 011007 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
/*
$Id: demo.c,v 1.8 2002/03/25 19:36:44 kripke Exp $
demo main file
*/
#include /* NULL */
#include /* printf */
#include /* atol */
#include "openisis.h"
int main ( int argc, const char **argv )
{
long rowid = atol(argv[argc-1]);
int db, i;
OpenIsisRec *rec;
if ( 0L == rowid ) {
printf( "usage: demo -db \n\tprints row with number \n" );
return 1;
}
openIsisLog( 0, NULL );
db = openIsisOpen( NULL, argv, argc );
rec = openIsisReadRow( db, rowid );
for ( i=0; ilen; i++ )
printf( "<%ld>%s\n",
rec->field[i].tag,
openIsis2Html(rec->field[i].val, rec->field[i].len),
rec->field[i].tag );
return 0;
} /* demo main */
./demo.pl 0100755 0001130 0001130 00000003201 07450672013 011154 0 ustar krip krip #!/usr/bin/perl
#/*
# openisis - an open implementation of the ISIS database
# Version 0.8.x (microversion see file Version)
# Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# This software is dedicated to the memory of Eckart Dietrich.
#
# This software is inspired by (but contains no code of) the iAPI
# Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
# See iAPI.txt for what it contains.
#EOH */
#
# $Id: demo.pl,v 1.8 2002/03/28 19:50:35 kripke Exp $
# demo script for openisis perl XSUB
#
use OpenIsis;
OpenIsis::log(0,'');
$db = OpenIsis::open( $ARGV[0] || 'db/cds/cds' );
for (1..OpenIsis::maxRowid( $db )) {
$row = OpenIsis::read( $db, $_ );
for (keys(%$row)) {
my $flds = $row->{$_};
for $f (@$flds) {
print "<$_>$f\n";
if ( $f =~ /^\^/ ) { # split subfields
$sub = OpenIsis::subfields( $f );
for $s (keys(%$sub)) {
print "<$_$s>$sub->{$s}\n";
}
}
}
}
}
./iAPI.txt 0100755 0001130 0001130 00000120551 07355467174 011244 0 ustar krip krip /*
this file composed by Klaus Ripke from the iAPI files by Robert Janusz.
this is the test app "ix.c" with all the includes slurped in,
see // #include "isomething.c"
you may run this with:
cp iAPI.txt iAPI.c
gcc iAPI.c
./a.out
*/
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
iAPI software: The Example of ISIS (UNIX v.3) Database Handle
Copyright (C) 2000 by Robert Janusz
E-mail: rj@jezuici.krakow.pl
Address: Robert Janusz, ul. Kopernika 26, 31-501 Krakow, Poland
tel: (0048-12) 4294416/432 ; fax: (0048-12) 4295003.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
// Linux gcc Compiler conventions !
#include
#include
#include
#include
// Global variables +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
static
char DBpath [71];
char UpCase [256];
#define NFIND 3 // let's keep it for future
#define PACKED __attribute__((packed)) // it shoud work on Linux gcc ;-)
// #include "idef.c"
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
iAPI software: The Type Definitions for Unix-ISIS databases
Copyright (C) 2000 by Robert Janusz
E-mail: rj@jezuici.krakow.pl
Address: Robert Janusz, ul. Kopernika 26, 31-501 Krakow, Poland
tel: (0048-12) 4294416/432 ; fax: (0048-12) 4295003.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
// gcc Linux Compiler conventions !
// *** ONLY UNIX ISIS = the difference between DOS/UNIX structures
// *************************** TYPE DEFINITIONS *****************************
// Cross Reference File XRF structure +++++++++++++++++++++++++++++++++++++++
typedef /* Packed */ struct // /* Packed */ and order are necessary for block reading
{
long int xpos PACKED;
// Block nr: 1, 2...; < 0: last block
long int xrec [127] PACKED;
// References to Master File records...
// xrec [...] = 2048 *xrmfb [21 bit] & xrmfp [11 bit]
// rec. deleted log. < 0 & > 0 (restore: -xrmfb)
// rec. deleted phis. =-1 & = 0 (ref@MST.ctrl)
// rec. does not exist = 0 & = 0
// new rec. to invert bit 1024 = 1
// rec. edited to inv. (add & del post.) bit 512 = 1
}
xrfBlock;
// Master File (MST) record structures... +++++++++++++++++++++++++++++++++++
typedef /* Packed */ struct // The first record in Master File
{
long int ctlmfn PACKED // Always 0
, nxtmfn PACKED // Next Master File Number
, nxtmfb PACKED; // Next block in Master File, the 1st has 1
short int nxtmfp PACKED // Next position in the last block
, mftype PACKED; // 0 for user's bases; 1 for messages
long int reccnt PACKED // Reserved for future
, mfcxx1 PACKED // Reserved for future
, mfcxx2 PACKED // LAN, > 0: inversion, update not possible
, mfcxx3 PACKED; // LAN, = 1: exists exclusive user
}
mstControl;
typedef /* Packed */ struct // Fixed part of record header that must be...
{ // included in the same block;
// the header can begin only on 0, 2, 4...498
long int mfn PACKED; // MFN number
short int mfrl PACKED; // Length of record (allways eaven)...
// LAN: < 0 ==> record blocked
long int mfbwb PACKED; // Block Pointer | If the record waits for inv...
short int mfbwp PACKED; // Position Pointer | ...ref. to old location
// mfbwb, mfbwp: = 0 at creation;
// after modif. = ref to the old record.
// xrmfp bit 1024 = 1 at creation
// (= waits to be inverted);
// xrmfp bit 512 = 1 at edit
// (=waits to be inv. & delete old references)
}
mstHdrFix;
typedef /* Packed */ struct
{
short int base PACKED // Position of data fields in the record
, nvf PACKED // Number of fields in the record
, status PACKED; // 0 = active, 1 = deleted logicaly (- xrfb < 0)
}
mstHdrTxt;
typedef /* Packed */ struct
{
mstHdrFix mhf PACKED; // This header is allways in MST file block
mstHdrTxt mht PACKED;
}
RecHeaders;
typedef /* Packed */ struct // Index for every field in the record
{ // dimension = 6 *nvf; base = 18 + 6 *nvf
short int tag PACKED // Label of the field
, pos PACKED // Position of the first char in data part: 0, 1...
, len PACKED; // Length of the field
}
mstIndex;
// Inverted File structures +++++++++++++++++++++++++++++++++++++++++++++++++
// Control File (CNT) structure..............................................
typedef /* Packed */ struct
{
short int idtype PACKED // B*tree type; 1 = N01/L01, 2 = N02/L02
, ordn PACKED // = 5; a half of size of key-record in N0x
, ordf PACKED // = 5; a half of size of key-record in L0x
, n PACKED // = 15; number of buffers for nodes
, k PACKED // = 5; number of buffers for 1st index level (k < n)
, lev PACKED; // Current number of levels in index; -1: no N0x
long int posrx PACKED // Reference to the root in N0x
, nmaxpos PACKED // Next record in N0x
, fmaxpos PACKED; // Next record in L0x
short int abnormal PACKED; // 0: N0x has only the root
}
cntBlock;
// Index of Dictionary (N0x) structures......................................
typedef /* Packed */ struct
{
long int pos PACKED; // Number of record
short int ock PACKED // Number of active keys in the record: 1, 2, 3...2*ordn
, it PACKED; // B*tree type
}
nodeHeader;
typedef /* Packed */ struct
{
char key [10] PACKED;
long int ref PACKED; // > 0: reference to next node;
// < 0: ref. to leaf: idx [1st].key = key
// = 0: inactive
}
nodeIndex1;
typedef /* Packed */ struct
{
char key [30] PACKED;
long int ref PACKED;
}
nodeIndex2;
typedef /* Packed */ struct
{
nodeHeader hdr PACKED;
nodeIndex1 idx [10] PACKED; // Vector of 2*ordn elements
}
nodeBlock1;
typedef /* Packed */ struct
{
nodeHeader hdr PACKED;
nodeIndex2 idx [10] PACKED; // Vector of 2*ordn elements
}
nodeBlock2;
typedef /* Packed */ union
{
nodeBlock1 nb1 PACKED;
nodeBlock2 nb2 PACKED;
}
nodeBlock;
// Leaf File (L0x) structures (all index terms)..............................
typedef /* Packed */ struct
{
long int pos PACKED; // Number of leaf record: 1, 2...
short int ock PACKED // Number of active keys in the record: 1..2*ordf
, it PACKED; // = 1 for L01; = 2 for L02
long int ps PACKED; // Next record in the order: key [ock] < ps^ idx [1st].key
}
leafHeader;
typedef /* Packed */ struct
{
char key [10] PACKED;
short int xxx PACKED; // *** ONLY UNIX ISIS;
long int infb PACKED // Reference to IFP segment *** != ISIS.MANUAL ***
, infp PACKED;
}
leafKey1;
typedef /* Packed */ struct
{
char key [30] PACKED;
short int xxx PACKED; // *** ONLY UNIX ISIS;
long int infb PACKED // Reference to IFP segment *** != ISIS.MANUAL ***
, infp PACKED;
}
leafKey2;
typedef /* Packed */ struct
{
leafHeader hdr PACKED;
leafKey1 idx [10] PACKED; // 2*ordf keys
}
leafBlock1;
typedef /* Packed */ struct
{
leafHeader hdr PACKED;
leafKey2 idx [10] PACKED; // 2*ordf keys
}
leafBlock2;
typedef /* Packed */ union
{
leafBlock1 lb1 PACKED;
leafBlock2 lb2 PACKED;
}
leafBlock;
// Inverted File Posting structures..........................................
// ordained postings organized in segments: Index --> Master File
typedef /* Packed */ struct
{
long int ifblk PACKED // Number of block
, ifrec [127] PACKED; // Vector of long int *** != ISIS.MANUAL ***
// In 1st block ifrec [1st/2nd] = next pos. in IFP
}
ifpBlock;
typedef /* Packed */ struct // This header and 1st posting - always in IFP block
{
long int ifpnxtb PACKED // Pointer to the next segment: 1, 2... | = 0: last seg.
, ifpnxtp PACKED // 0, 1... | = 0: last seg.
, ifptotp PACKED // Tot. number of postings (o.k. only in the 1st segment)
// = Sum (segment=1, nseg; ifpsegp)
, ifpsegp PACKED // Number of postings in this segment
, ifpsegc PACKED; // Max. number of postings in this segment (<= 32768)
}
ifpHeader;
// ifpHeader & 1st ifpPosting always in the same ifpBlock
// Postings in Segment are always in chain of IFP blocks
typedef /* Packed */ struct // Stored left-->right + 0 --> 8 char
{
long int pmfn PACKED; // First 3 bytes --> Master File
short int ptag PACKED; // Field ident - according to File Selection Table (FST)
char pocc PACKED; // Number of occurence of the field
short int pcnt PACKED; // Counter of the termin inside the field
}
ifpPosting;
typedef struct
{
int opened; // Which files are opened (bit spec.)...
FILE *mst; // 1
FILE *xrf; // 2
FILE *ifp; // 4
FILE *n01; // 8
FILE *l01; // 16
FILE *n02; // 32
FILE *l02; // 64
mstControl mctrl; // MST ctrl record
cntBlock cnt1, cnt2; // Inv. Files ctrl records
long int ibm, ipm; // IFP new Segment Bock/Pos
char name [72]; // Database name
}
BaseCtrl;
typedef struct // This structer must be allways corelated with BaseCtrl
{
char st [31]; // Max. string to find
short int v; // Field to find; 0: all fields
short int it; // Type of B*Tree <== strlen (st)
leafBlock lbl; // Block L0x with index terms
short int ilk; // Position of term in lbl...idx
char *term; // The term in lbl...idx
long int ib, ip; // Number of block/position for IFP
ifpBlock ibl; // Block IFP with posting/s
ifpHeader ihd; // Header for Segment of postings
long int iiseg; // Number of posting in current Segment (index)
long int iitot; // Number of posting in total (index)
long int iall; // Number of posting in all Segments
char *ipst; // Ref. to last posting string [8 chars]
cntBlock *cntr; // BaseCtrl.cntr1/2 <== it
FILE *n0x; // N01 or N02 <== it, = BaseCtrl.n0x
FILE *l0x; // L01 or L02 <== it, = BaseCtrl.l0x
}
TableQuery;
#define RECLENGTH 8192
typedef union
{
char buf [RECLENGTH];
RecHeaders hdrs;
}
RecCtrl;
// *** MAIN DATABASE FILE RELATIONS (MST <--> XRF) ***
// 1 2 3
// MST |-mblk-|-mblk-|-mblk-|-mblk-|-mblk-|-mblk-|-mblk-|-mblk-|...
// ctrl |----record----|
// |--->|
// 1 2 3 | pointer: B/P (P in [0, 2, 4... 498])
// XRF |-xblk-|-xblk-|-xblk-|...
// MFN= 1... 128... 255...
// Record |-->-pos->--|
// |-mhf-mht-|-idx[1]...idx[nvf]-|---text...---|
// |-------------- record buffer --------------|
// *** INVERTED FILE RELATIONS (N0x <--> L0x <-->IFP) ***
// N0x |-nbl-|-nbl-|-nbl-|-nbl-|-nbl-|-nbl-|...
// -->-root->-| |-->--next-->--| |->-->-| r<0
// |
// L0x |--lbl--|--lbl--|--lbl--|--lbl--|--lbl--|--lbl--|...
// |
// |-<--------------<-------------<-| r<0
// *** POSTING *** |
// IFP |---ibl---|---ibl---|---ibl---|---ibl---|---ibl---|---ibl---|...
// |next |-ihd-|---Segment---| |-ihd-|---Segment---|
// |Seg. |-->------>------>-next->--| |||...
// MFN...
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// #include "idef.c"
// Let's specify non int header ;-) just to keep order
void ShowRec (RecCtrl *rc);
void setQuery (BaseCtrl *bc, TableQuery *tq, char *sx, short int fld);
void isis ();
// #include "iutil.c"
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
iAPI software: The UTIL procedures for Unix-ISIS databases
Copyright (C) 2000 by Robert Janusz
E-mail: rj@jezuici.krakow.pl
Address: Robert Janusz, ul. Kopernika 26, 31-501 Krakow, Poland
tel: (0048-12) 4294416/432 ; fax: (0048-12) 4295003.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
// gcc Linux Compiler conventions !
// *** ONLY UNIX ISIS = the difference between DOS/UNIX structures
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int ReadMem (void *ptr, FILE *f, long int nr, long int size)
{ // Read "nr"'s block of size "size" from file "f" into "ptr"...
if (fseek (f, nr * size, SEEK_SET) != 0) return (-1);
if (fread (ptr, size, 1, f) == 0) return (-2);
return (0); // O.K.
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void doUpcase (char *s, char *st, int slen)
{ int i;
for (i = 0; i < slen; i++)
{ if (i < strlen (st)) s [i] = UpCase [(int) st [i]]; else s [i] = ' '; }
s [slen] = 0;
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int sComp (char *s1, char *s2, int len)
{ int i;
for (i = 0; i < len; i++)
{ if (s1 [i] > s2 [i]) return (1); // .gt.
if (s1 [i] < s2 [i]) return (-1); // .lt.
}
return (0); // .eq.
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int bfOpen (FILE **f, BaseCtrl *bc, char *ext, char *opt)
{ char st [81];
if (strlen (bc->name) + strlen (ext) > 80) return (-2);
strcpy (st, bc->name); strcat (st, ext);
if ((*f = fopen (st, opt)) == NULL) return (-1);
return (0);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int InitBase (BaseCtrl *bc)
// Error return: -1 base file/s; -2 inv. file/s; -3 mst. ctrl rec/s;
// -4 cnt param/s; -5 ifp param/s
{ ifpBlock ib;
bc->opened = 0;
if (bfOpen (&bc->mst, bc, ".MST", "rb") != 0) return (-1);
bc->opened += 1; // MST opened
if (bfOpen (&bc->xrf, bc, ".XRF", "rb") != 0) return (-1);
bc->opened += 2; // XRF opened
if (bfOpen (&bc->ifp, bc, ".CNT", "rb") != 0) return (-2);
if (fread (&bc->cnt1, sizeof (cntBlock), 1, bc->ifp) == 0 ||
fread (&bc->cnt2, sizeof (cntBlock), 1, bc->ifp) == 0) return (-2);
if (fclose (bc->ifp) != 0) return (-3);
if (bfOpen (&bc->ifp, bc, ".IFP", "rb") != 0) return (-2);
bc->opened += 4; // IFP opened
if (bfOpen (&bc->n01, bc, ".N01", "rb") != 0) return (-2);
bc->opened += 8; // N01 opened
if (bfOpen (&bc->l01, bc, ".L01", "rb") != 0) return (-2);
bc->opened += 16; // L01 opened
if (bfOpen (&bc->n02, bc, ".N02", "rb") != 0) return (-2);
bc->opened += 32; // N02 opened
if (bfOpen (&bc->l02, bc, ".L02", "rb") != 0) return (-2);
bc->opened += 64; // L02 opened
if (fread (&bc->mctrl, sizeof (mstControl), 1, bc->mst) == 0) return (-3);
if (bc->cnt1.ordn != 5 || bc->cnt1.ordf != 5 || bc->cnt1.n != 15 ||
bc->cnt1.k != 5 || bc->cnt2.ordn != 5 || bc->cnt2.ordf != 5 ||
bc->cnt2.n != 15 || bc->cnt2.k != 5) return (-4);
if (fread (&ib, sizeof (ifpBlock), 1, bc->ifp) == 0) return (-5);
if (ib.ifblk != 1) return (-5);
bc->ibm = ib.ifrec [0]; bc->ipm = ib.ifrec [1];
return (0);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void Done (BaseCtrl *bc) // Close the database files
{
if (bc->opened && 1) fclose (bc->mst);
if (bc->opened && 2) fclose (bc->xrf);
if (bc->opened && 4) fclose (bc->ifp);
if (bc->opened && 8) fclose (bc->n01);
if (bc->opened && 16) fclose (bc->l01);
if (bc->opened && 32) fclose (bc->n02);
if (bc->opened && 64) fclose (bc->l02);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
char UpMe (char c) // ^A = ^a etc.
{
if (c > 96) c = c - 32;
return (c);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void outS (char *st)
{ char i;
if (st == NULL) return;
for (i = 0; i < strlen (st); i++)
// c = tblconvert [st [i]];
if (st [i] == (char) 145) printf ("<"); // This is HTML <
else if (st [i] == (char) 146) printf (">"); // and that is >
else printf ("%c", st [i]);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void outC (char ch)
{ char st [2];
st [0] = ch; st [1] = 0; outS (st);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void outD (double x, int n)
{ char s [80];
gcvt (x, n, s);
outS (s);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void outN (long int x) // long int -> sting @ base=10
{
// char st [33];
// ltoa (x, st, 10); outS (st);
printf ("%li", x);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// #include "imasterf.c"
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
iAPI software: The Master File access to Unix-ISIS databases
Copyright (C) 2000 by Robert Janusz
E-mail: rj@jezuici.krakow.pl
Address: Robert Janusz, ul. Kopernika 26, 31-501 Krakow, Poland
tel: (0048-12) 4294416/432 ; fax: (0048-12) 4295003.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
// gcc Linux Compiler conventions !
// *** ONLY UNIX ISIS = the difference between DOS/UNIX structures
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int GetRec (BaseCtrl *bc, long int mfnr, RecCtrl *rc, int bufvol)
{ char mblk [512]; // Master File block;
long int xmfb, xmfp; // Pointer to Master File (last read) record
// xMfB < 0 : record deleted (status = 1), -xMfB --> block;
// xMfP = 0: deleted phisicaly
// = 0 & = 0: does not exist
// 1st block: xMfB = 1, xMfP = 0; pointer = xMfB * 2048 + xMfP
// ==> mst < 500 MBy;
long int xn, xb;
int ret, nch;
xrfBlock xblk;
mstHdrFix *hf = &rc->hdrs.mhf;
mstHdrTxt *ht = &rc->hdrs.mht;
char *b = rc->buf;
if ((mfnr < 1) || (mfnr >= bc->mctrl.nxtmfn)) return (-2); // No record
// Find the address in XRF...
xn = mfnr - 1; xb = xn / 127;
if (ReadMem (&xblk, bc->xrf, xb, sizeof (xrfBlock)) != 0) return (-1);
xn = xblk.xrec [xn % 127]; // = address in MST
xmfb = xn / 2048; // Master File Block
xmfp = xn % 512; // Master File Position
// Find the record in MST...
if (xmfb < 1) return (1); // Record deleted
ret = 0; // Record o.k.
if (ReadMem (&mblk, bc->mst, xmfb - 1, sizeof (mblk)) != 0) return (-1);
// Copy headers into record buffer...
memcpy (b, &mblk [xmfp], sizeof (mstHdrFix));
if (hf->mfn != mfnr) return (-2); // Database corrupted
if (hf->mfrl >= bufvol) return (-3); // Buffer too small
// Get the rest of the record
xmfp += sizeof (mstHdrFix); // Posinion in block
xb = sizeof (mstHdrFix); // Position in buf
xn = labs (hf->mfrl) - xb; // Number of chars to read
if (xn <= 0) return (4); // No fields ??
for ( ; ; ) // We have some chars
{ // Find number of chars to copy from this block
if (xn <= (nch = sizeof (mblk) - xmfp)) nch = xn;
memcpy (b + xb, &mblk [xmfp], nch);
xb += nch; xn -= nch; xmfp = 0;
if (xn <= 0) break; // Nothing more to read
if (fread (&mblk, sizeof (mblk), 1, bc->mst) == 0) return (-1);
}
if (ht->status != 0) ret = 1; // Record deleted logicaly
if (hf->mfbwb != 0 || hf->mfbwp != 0) ret = 2; // Record not inv.
if (hf->mfrl < 0) ret = 3; // Record blocked
return (ret);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int IsF (RecCtrl *rc, short int v, char ch, char *st)
{ short int n, k, kk, flag;
char *b = rc->buf;
mstHdrTxt *mht = &rc->hdrs.mht;
mstIndex *ind = (mstIndex *) &rc->buf [sizeof (RecHeaders)];
flag = 0;
for (n = 0; n < mht->nvf; n++)
{
if (ind->tag == v) // there is a tag
{
if (ch == ' ') // field found
{ flag = 1; break; }
else // there is sub-field
{ flag = 1;
kk = 0; k = mht->base + ind->pos;
while (b [k] != '^' || b [k + 1] != ch) // convention ;-)
{ k += 1; kk += 1;
if (kk >= ind->len) // out of field
return (0);
}
break;
}
}
ind += 1; // next index ref.
}
if (flag == 0) return (0); // false
if (st == NULL) return (1); // true
else
if (sComp (&b [mht->base + ind->pos], st, ind->len) == 0)
return (1);
else return (0);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void RepF (RecCtrl *rc, char nf, ...)
// Repeated "nf" fields in record "rc", the left label appears only if:
// - begins with "\n", - there was another field and there is defined
// filed; the right label appears only if ther was defined field.
{ va_list pinfo; // for ... parameter list
int i, c, flag, n, k, kk, v;
int hit = 0;
char *sl, *sr;
char *b = rc->buf;
mstHdrTxt *mht = &rc->hdrs.mht;
mstIndex *ind = (mstIndex *) &rc->buf [sizeof (RecHeaders)];
for (n = 0; n < mht->nvf; n++)
{
va_start (pinfo, nf);
for (i = 0; i < nf; i++)
{
kk = 0;
k = mht->base + ind->pos;
sl = va_arg (pinfo, char *); // left
v = va_arg (pinfo, short int); // field
c = va_arg (pinfo, char); // sub-field
c = UpMe (c);
sr = va_arg (pinfo, char *); // right
if (v != ind->tag) // another Tag
break;
if (c != ' ') // sub-field
{ flag = 1;
while (b [k] != '^' || UpMe (b [k + 1]) != c) // convention ;-)
{ k += 1; kk += 1;
if (kk >= ind->len)
{ flag = 0; break; }
}
if (flag) { k += 2; kk += 2; }
}
else flag = 1;
if (flag)
{ // Reference to NULL = NW error !!!
if (sl == NULL) flag = 0;
else if (*sl == '\n') flag = 1; else flag = 0;
if (hit != 0 || flag) outS (sl);
hit += 1;
if (c == ' ')
while (kk < ind->len)
{
if (b [k] != '<' && b [k] != '>') // HTML special ;-)
outC (b [k]);
k += 1; kk += 1;
}
else
while (kk < ind->len && b [k] != '^') // convention ;-)
{
if (b [k] != '<' && b [k] != '>') // HTML special ;-)
outC (b [k]);
k += 1; kk += 1;
}
outS (sr);
}
}
va_end (pinfo);
ind += 1;
}
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// #include "iinvterm.c"
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
iAPI software: The Inv. File Terms access for Unix-ISIS databases
Copyright (C) 2000 by Robert Janusz
E-mail: rj@jezuici.krakow.pl
Address: Robert Janusz, ul. Kopernika 26, 31-501 Krakow, Poland
tel: (0048-12) 4294416/432 ; fax: (0048-12) 4295003.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
// gcc Linux Compiler conventions !
// *** ONLY UNIX ISIS = the difference between DOS/UNIX structures
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int FindFirst (TableQuery *tq)
{
short int i, j // Indexes for idx []
, srec, klen, sidx; // Size (rec), key length, size (idx)
long int r; // Reference (record number); < 0: --> L0x --> IFP
char *ref;
leafHeader *lhdr = &tq->lbl.lb1.hdr; // = ...lb2.hdr
nodeBlock nbl;
nodeHeader *nhdr = &nbl.nb1.hdr; // = ...nb2.hdr
if (tq->it == 1)
{ klen = 10; srec = sizeof (nodeBlock1); sidx = sizeof (nodeIndex1); }
else
{ klen = 30; srec = sizeof (nodeBlock2); sidx = sizeof (nodeIndex2); }
r = tq->cntr->posrx; // Root in N0x
while (r > 0L)
{
if (ReadMem (&nbl, tq->n0x, r - 1, srec) != 0) return (-1);
j = nhdr->ock;
if (nhdr->pos != r || nhdr->it != tq->it ||
j < 1 || j > 10) return (-3); // 10 = 2*ordn
ref = (char *) &nbl.nb1.idx; // Reference to first elem. of index
ref += sidx; // The first key is allways <= st, skip it
for (i = 1; i < j; i++)
if (sComp (ref, tq->st, klen) > 0) break;
else ref += sidx;
ref -= sidx; // Correct value
r = *(long int *) (ref + klen); // We have the "ref" value
if (r > tq->cntr->nmaxpos) return (-3); // Out of N0x
}
// Now find the term in L0x index
if (tq->it == 1)
{ srec = sizeof (leafBlock1); sidx = sizeof(leafKey1); }
else
{ srec = sizeof(leafBlock2); sidx = sizeof(leafKey2); }
r = - r;
if (ReadMem (&tq->lbl, tq->l0x, r - 1, srec) != 0) return (-1);
j = lhdr->ock;
if (lhdr->pos != r || lhdr->it != tq->it ||
j < 1 || j > 10) return (-2); // 10 = 2*ordf
tq->ilk = 0;
ref = (char *) &tq->lbl.lb1.idx; // Reference to first elem. of index
ref += sidx; // The first key is allways <= s, skip it
for (i = 1; i < j; i++)
if (sComp (ref, tq->st, klen) > 0) break;
else { ref += sidx; tq->ilk += 1; }
ref -= sidx;
tq->term = ref; // We have the term address
if (tq->it == 1)
ref += 12; // += 10; *** NOT UNIX
else ref += 32; // += 30; *** NOT UNIX
tq->ib = *(long int *) ref; // = infb
ref += sizeof (long int);
tq->ip = *(long int *) ref; // = infp
return (0); // O.K.
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int FindNext (TableQuery *tq)
{ int srec; // Block size
long int r; // Reference (record number); < 0: --> L0x --> IFP
leafHeader *lhdr = &tq->lbl.lb1.hdr; // = ...lb2.hdr
if (tq->it == 1) srec = sizeof (leafBlock1);
else srec = sizeof (leafBlock2);
tq->ilk += 1; // Next key in the index
if (tq->ilk >= lhdr->ock) // Out of index
{ r = lhdr->ps; // Next L0x record in order
if (r <= 0)
{ tq->ilk -= 1; return (1); } // Nothing more, end of index
if (ReadMem (&tq->lbl, tq->l0x, r - 1, srec) != 0) return (-1);
if (lhdr->pos != r || lhdr->it != tq->it ||
lhdr->ock < 1 || lhdr->ock > 10) return (-2); // 10 = 2*ordf
tq->ilk = 0;
}
srec = tq->ilk;
if (tq->it == 1)
{ tq->term = tq->lbl.lb1.idx [srec].key;
tq->ib = tq->lbl.lb1.idx [srec].infb;
tq->ip = tq->lbl.lb1.idx [srec].infp;
}
else
{ tq->term = tq->lbl.lb2.idx [srec].key;
tq->ib = tq->lbl.lb2.idx [srec].infb;
tq->ip = tq->lbl.lb2.idx [srec].infp;
}
return (0); // We have the new key
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int Find (BaseCtrl *bc, TableQuery tabq [], int ifind, long int *npg
, RecCtrl *rc, long int recvol)
{ int i;
if (ifind <= 0 || ifind > NFIND) return (-1);
for (i = 0; i < ifind; i++)
if (FindFirst (&tabq [i]) < 0) return (-1);
i = Postings (bc, tabq, ifind, npg, rc, recvol);
return (i);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int IndexPrep (BaseCtrl *bc, TableQuery *tq1, TableQuery *tq2, char *st)
{ char sx [31]; // tqx->v contains the status of L0x index
int i;
if (st == NULL)
for (i = 0; i < 30; i++) sx [i] = ' ';
else
for (i = 0; i < 30; i++)
if (i < strlen (st)) sx [i] = st [i]; else sx [i] = ' ';
sx [30] = 0;
setQuery (bc, tq2, sx, 0); // We have a long term in tq2; v := 0
if (FindFirst (tq2) < 0) return (-1);
sx [10] = 0;
setQuery (bc, tq1, sx, 0); // We have a short term in tq1; v := 0
if (FindFirst (tq1) < 0) return (-1);
while (sComp (tq1->term, tq1->st, 10) < 0)
if ((tq1->v = FindNext (tq1)) != 0) break; // At the end or error
if (tq1->v < 0) return (-1);
while (sComp (tq2->term, tq2->st, 30) < 0)
if ((tq2->v = FindNext (tq2)) != 0) break; // At the end or error
if (tq2->v < 0) return (-1);
return (IndexTest (tq1, tq2));
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int IndexTest (TableQuery *tq1, TableQuery *tq2)
{
if (tq1->v == 1) // At the end of 1st
if (tq2->v == 1) // At the end of 2nd
return (0); // All done
else return (2); // We have only 2nd
else
if (tq2->v == 1) return (1); // We have only 1st
else if (sComp (tq1->term, tq2->term, 10) <= 0) return (1);
else return (2);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int IndexTerm (int i, TableQuery *tq1, TableQuery *tq2)
{
if (i == 1) tq1->v = FindNext (tq1); // The term was in 1st
else if (i == 2) tq2->v = FindNext (tq2); // The term was in 2nd
else return (-1);
return (IndexTest (tq1, tq2));
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void setQuery (BaseCtrl *bc, TableQuery *tq, char *sx, short int fld)
{ // Must be activated AFTER the initialization of database
tq->v = fld;
if (strlen (sx) <= 10)
{ tq->it = 1; doUpcase (tq->st, sx, 10);
tq->cntr = &bc->cnt1; tq->n0x = bc->n01; tq->l0x = bc->l01;
}
else
{ tq->it = 2; doUpcase (tq->st, sx, 30);
tq->cntr = &bc->cnt2; tq->n0x = bc->n02; tq->l0x = bc->l02;
}
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// #include "ipost.c"
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
iAPI software: The Inv. File Postings access for Unix-ISIS databases
Copyright (C) 2000 by Robert Janusz
E-mail: rj@jezuici.krakow.pl
Address: Robert Janusz, ul. Kopernika 26, 31-501 Krakow, Poland
tel: (0048-12) 4294416/432 ; fax: (0048-12) 4295003.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
// gcc Linux Compiler conventions !
// *** ONLY UNIX ISIS = the difference between DOS/UNIX structures
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int PostSeg (BaseCtrl *bc, TableQuery *tq)
{
if (ReadMem (&tq->ibl, bc->ifp, tq->ib - 1, sizeof (ifpBlock)) != 0)
return (-1);
if (labs (tq->ibl.ifblk) - tq->ib != 0) return (-2);
memcpy (&tq->ihd, &tq->ibl.ifrec [tq->ip], sizeof (ifpHeader));
tq->ip += 3; // = long int.s / (header - posting) ***!= ISIS.MANUAL ***
tq->iiseg = 0;
return (0);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int PostNext (BaseCtrl *bc, TableQuery *tq)
{
if (tq->iitot >= tq->iall) return (1); // No more postings
if (tq->iiseg >= tq->ihd.ifpsegp) // No more postings in this seg.
{ // Get next Segment...
tq->ib = tq->ihd.ifpnxtb; tq->ip = tq->ihd.ifpnxtp;
if (PostSeg (bc, tq) != 0) return (-2);
}
tq->iiseg += 1; tq->iitot += 1;
tq->ip += 2; // = Number of long int / posting ***!= ISIS.MANUAL ***
if (tq->ip > 125) // ***!= ISIS.MANUAL ***
{ // Get next block for this segment; we HAVE TO find the right block!
tq->ib += 1; tq->ip = 0;
if (ReadMem (&tq->ibl, bc->ifp, tq->ib - 1, sizeof (ifpBlock)) != 0)
return (-1);
if (labs (tq->ibl.ifblk) != tq->ib) return (-3);
}
tq->ipst = (char *) &tq->ibl.ifrec [tq->ip];
return (0);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void PostMake (long int mfn, short int tag, char occ, short int pcnt
, char *s, char *slen)
{ ifpPosting rnp;
char *p = (char *) &rnp;
rnp.pmfn = mfn; rnp.ptag = tag; rnp.pocc = occ; rnp.pcnt = pcnt;
if (tag == 0) *slen = 3; // Only MFN
else *slen = 5; // MFN and TAG
s [2] = p [0]; s [1] = p [1]; s [0] = p [2]; // = pmfn
s [4] = p [4]; s [3] = p [5]; s [5] = p [6]; s [7] = p [7]; s [6] = p [8];
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void PostInv (ifpPosting *ifpp, void *ps)
{ char *p = (char *) ifpp;
char *s = (char *) ps;
// Posting string: |0 a |1 b |2 c |3 d |4 e |5 f |6 g |7 h | ps *s
// ifpPosting: |0 c|1 b |2 a |3 0 |4 e |5 d |6 f |7 h |8 g | ifpp *p
// |pmfn |ptag |pocc|pcnt |
p [0] = s [2]; p [1] = s [1]; p [2] = s [0]; p [3] = 0; // = pmfn
p [4] = s [4]; p [5] = s [3]; p [6] = s [5]; p [7] = s [7]; p [8] = s [6];
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
int main (int argc, char **argv) // char *argv []
{ char c;
printf ("ix \n");
printf (" ::= UNIX-ISIS (v.3) MF without .MST\n");
printf ("ix ver. 0.3, Copyright (C) 2000 by Robert Janusz\n");
printf ("ix comes with ABSOLUTELY NO WARRANTY\n");
printf ("License: GPL; see http://www.gnu.org/ for details\n");
for (c = 'A'; c <= 'z'; c++) // Let's prepare it, we use API, not an example
UpCase [c] = UpMe (c);
if (argv [1] != NULL) {
strcpy (DBpath, argv [1]); // Database; no test ! ;-) it's only an example
isis ();
}
return (0);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void isis ()
{
int j, v, k;
long int nrc;
BaseCtrl bc;
RecCtrl rc;
char st [31];
TableQuery tabq [NFIND];
strcpy (bc.name, DBpath);
InitBase (&bc);
// ShowRec but not all...
printf ("\n*** ShowRec ***\n\n");
//for (nrc = 1L; nrc < bc.mctrl.nxtmfn; nrc++) {
for (nrc = 10L; nrc < 20L; nrc++) {
if ((j = GetRec (&bc, nrc, &rc, RECLENGTH)) == 0) {
ShowRec (&rc);
}
}
// ShowIndex
printf ("\n*** ShowIndex: %s ***\n\n", st);
strcpy (st, "TEST"); // just an Inv. File term
v = 0; // term counter
k = IndexPrep (&bc, &tabq [0], &tabq [1], st); // prepare indexes
while (k > 0) {
j = 20 * k - 10; // We have the term length ;-)
memcpy (&st, &tabq [k - 1].term [0], j); // keep it in st
st [j] = 0;
if ((v += 1) > 12) // we want 12 terms in a list
{
printf ("->%s\n", st); // the next term
break;
}
printf ("%s\n", st);
k = IndexTerm (k, &tabq [0], &tabq [1]); // Next term
}
// Find a term & show records
printf ("\n*** Search for: %s ***\n\n", st); // Let's use the last term
setQuery (&bc, &tabq [0], st, 0); // String to search in a filed 0 = any
j = Find (&bc, tabq, 1, &nrc, &rc, RECLENGTH); // 1 term to search for
// Clean all
printf ("\nO.K.\n");
Done (&bc);
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
void ShowRec (RecCtrl *rc)
{ long int nr;
short int n, k, kk;
char st [256];
char *b = rc->buf;
mstHdrTxt *mht = &rc->hdrs.mht;
mstIndex *ind = (mstIndex *) &rc->buf [sizeof (RecHeaders)];
nr = rc->hdrs.mhf.mfn; // Record header
printf ("=== %li\n", nr);
for (n = 0; n < mht->nvf; n++)
{ // Show fileds without formating
printf ("*** [%li] ", (long int) ind->tag);
k = mht->base + ind->pos;
for (kk = 0; kk < ind->len; kk++)
printf ("%c", b [k++]);
printf ("\n");
ind += 1; // Next index
}
printf ("___\n");
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// This procedure is prepared to do a simple AND operator after some
// extensions, so it can be localy a little strange ;-)
int Postings (BaseCtrl *bc, TableQuery tabq [], int ifind, long int *npg
, RecCtrl *rc, long int recvol)
{ int j, state;
long int ipg, rn;
ifpPosting ip;
TableQuery *tq;
if (ifind < 1) goto finish; // nothing to do
ipg = 1L; // Index for page break
{ tq = &tabq [0]; // Let's think only about one term to handle (example)
if (PostSeg (bc, tq) != 0) return (-1);
tq->iall = tq->ihd.ifptotp; // Only 1st Segment has tot. post. number
tq->iitot = 0;
if (PostNext (bc, tq) != 0) return (-1);
PostInv (&ip, tq->ipst);
if (sComp (tq->st, tq->term, 20 * tq->it - 10) != 0)
ipg = 0L; // the term not found
}
if (ipg != 1L) goto finish;
for ( ; ; ) // Global loop...
{
// Is there a field
{ tq = &tabq [0];
if (tq->v != 0)
while (ip.ptag != tq->v)
{
if (PostNext (bc, tq) > 0) goto finish; // No more postings
PostInv (&ip, tq->ipst);
}
}
rn = ip.pmfn;
state = 1;
if (ip.pmfn != rn)
state = 0;
if (state) // We have a record to show
{
if ((j = GetRec (bc, rn, rc, recvol)) == 0)
ShowRec (rc);
else
printf ("Record state = %li\n", j);
do // Shift all postings
{ tq = &tabq [0];
if (PostNext (bc, tq) > 0) goto finish; // No more postings
PostInv (&ip, tq->ipst);
}
while (ip.pmfn <= rn); // MFN equals
}
else // state == 0, Shift all postings
while (ip.pmfn < rn) // The last is to examin
{ tq = &tabq [0];
if (PostNext (bc, tq) > 0) goto finish; // No more postings
PostInv (&ip, tq->ipst);
}
}
finish:
return (0); // The output is finished
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
./ldb.c 0100644 0001130 0001130 00000135510 07550303061 010601 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
/*
$Id: ldb.c,v 1.27 2002/10/07 13:03:30 kripke Exp $
implementation of general db access functions.
*/
#include
#include
#include
#include
#include /* PATH_MAX */
#ifdef WIN32
# define LACKING_MMAP
#else
# include
#endif /* WIN32 */
/* special */
#if defined( __GNUC__ ) && defined ( alloca )
#include
#endif
#include "lstr.h"
#include "lio.h"
#include "ldb.h"
/* ************************************************************
private types
*/
/** extension of master file proper. */
static const char EXT_MST_MST[] = ".mst";
/** extension of master file xref. */
static const char EXT_MST_XRF[] = ".xrf";
typedef enum {
MST_MST,
MST_XRF,
MST_FILES
} mst_file;
static const char * const EXT_MST[MST_FILES] = {
EXT_MST_MST,
EXT_MST_XRF
};
/** extension of inverted file short term nodes. */
static const char EXT_INV_N01[] = ".n01";
/** extension of inverted file short term leaves. */
static const char EXT_INV_L01[] = ".l01";
/** extension of inverted file long term nodes. */
static const char EXT_INV_N02[] = ".n02";
/** extension of inverted file long term leaves. */
static const char EXT_INV_L02[] = ".l02";
/** extension of inverted file control records. */
static const char EXT_INV_CNT[] = ".cnt";
/** extension of inverted file postings. */
static const char EXT_INV_IFP[] = ".ifp";
typedef enum {
INV_N01,
INV_L01,
INV_N02,
INV_L02,
INV_CNT,
INV_IFP,
INV_FILES
} inv_file;
static const char * const EXT_INV[INV_FILES] = {
EXT_INV_N01,
EXT_INV_L01,
EXT_INV_N02,
EXT_INV_L02,
EXT_INV_CNT,
EXT_INV_IFP
};
typedef long lblk[128];
typedef struct _Db {
int dbid;
long flags;
const char *path;
const char *name;
int mst[MST_FILES]; /* master file */
int inv[INV_FILES]; /* primary inverted file */
long mfc[LMFC__FL]; /* master file control record */
unsigned long mflen; /* master file length */
long xrf[129]; /* last used xrf block : THREAD THREAT */
long xrlen; /* length of xrf (in blocks) */
lblk *mmap; /* memory map of xrf */
long mmlen; /* length of map (in blocks) */
long cnt[LDB_INDEXES][LCNT__FL]; /* two cnt records */
short tlen[LDB_INDEXES]; /* max term length for each index */
const char *format;
int set[2]; /* record sets, should be { LSET_MST, LSET_INV } */
LdbCTab ctab[2];
} Db;
/* db flags */
#define DB_OPEN 0x10000
#define DB_INVOPEN 0x20000
#define DB_WRITABLE 0x40000
#define DB_MODIFIED 0x80000
#define DB_MMAP 0x10
#define DB_VARI 0xf /* mask for variant */
/* get master or inverted record set */
#ifdef HAVE_NOSTANDARD_RSETS /* far future */
# define DB_RSET( db, s ) lstrlib[ (db)->set[s] ]
#else /* simply LSET_MST and LSET_INV */
# define DB_RSET( db, s ) lstrlib[ s ]
#endif
/* get xstr for record rec in set */
#define DB_XSTR( db, set, rec ) \
DB_RSET( db, set ).desc[ DB_VARI & (db)->flags ][ rec ]
/* get record names for record rec in set */
#define DB_RNAM( db, set, rec ) \
DB_RSET( db, set ).name[ rec ]
/** packed little endian masterfile control structure.
*/
typedef struct _Mfc {
long ctlm;
long nmfn;
long nmfb;
short nmfp;
short type;
long rcnt;
long mfx1;
long mfx2;
long mfx3;
} Mfc;
/** packed little endian masterfile record.
*/
typedef struct _Dict {
short tag;
short pos;
short len;
} Dict;
/** packed little endian masterfile record.
*/
typedef struct _Mfr {
long mfn;
short recl; /* a.k.a. mfrl */
short bwbl; /* low part of long */
short bwbh; /* high part of long */
short bwp;
/* it is believed, that this first five fields up to here (12 bytes packed)
are to be in one 512-byte block; the manual mentiones even 14 bytes ... ???
*/
short base;
short nvf;
short stat;
Dict dict[1];
} Mfr;
/* ************************************************************
private data
*/
static Db defdbspace[32];
/* array of open dbs. should expand dynamically. */
static Db *dbs = defdbspace;
static int dbs_len = sizeof(defdbspace)/sizeof(defdbspace[0]);
static OpenIsisLockFunc *lock = NULL;
#define LOCK() (NULL != lock && lock(1))
#define UNLOCK() do { \
if ( NULL != lock && lock(0) ) \
log_msg( LOG_SYSERR, "could not unlock %d", __LINE__ ); \
} while (0)
/* ************************************************************
private functions
*/
static Db *getDb ( int id )
{
if ( 0 <= id && id < dbs_len && dbs[id].flags )
return &dbs[id];
log_msg( LOG_ERROR, "attempt to access bad db id %d", id );
return NULL;
} /* getDb */
/* ************************************************************
start of io section
*/
static int openfiles ( int *fid, const char *path, const char *name,
const char * const *ext, int nfiles, int writable )
{
int plen = NULL==path ? 0 : strlen(path);
int nlen = strlen(name);
char filename[ PATH_MAX ];
int i,ret;
char *e;
if ( sizeof(filename) < (unsigned int)(plen+1+nlen+4+1) ) {
log_msg( LOG_ERROR, "path(%d) or filename(%d) too long '%.30s...'",
plen, nlen, filename );
return -LERR_FAULT;
}
if ( plen ) {
memcpy( filename, path, plen );
filename[plen++] = '/';
}
memcpy( filename+plen, name, nlen );
e = filename + plen+nlen;
e[4] = 0;
for ( i=0; i got )
return got;
#ifndef NDEBUG
if ( LOG_DO( LOG_ALL ) )
LOG_HEX( dst, got );
#endif
if ( siz == (int)got )
return 0;
log_msg( LOG_WARN, "got %u bytes wanted %d at %ld in %d",
got, siz, where, fid );
return 1+(int)got;
} /* readblk */
/* ************************************************************
end of io section
*/
static long *nrec ( long *xstr )
{
long *dst = (long*)lmalloc( xstr[LSTR_ILEN] );
if ( NULL != dst )
*dst = *xstr;
return dst;
} /* nrec */
typedef struct _b8 { char x[8]; } b8;
typedef struct _b4 { char x[4]; } b4;
typedef struct _b2 { char x[2]; } b2;
static int convert ( long *dst, char *src, long *xstr )
{
static int pow2[] = { 1, 2, 4, 8 };
int occ = -1;
long *xmbrs = xstr+LSTR_XMBR;
int nmbrs = LSTRFIX(*xstr);
long *mbr;
char *buf, *part = src, *srcend = src + xstr[LSTR_XLEN];
if ( NULL == dst ) {
log_msg( LOG_SYSERR, "out of memory (no dst) in convert" );
return -LERR_NOMEM;
}
if ( LSTRLEN(*xstr) > xstr[LSTR_ILEN] ) {
log_msg( LOG_ERROR, "bad ilen %ld need %ld",
xstr[LSTR_ILEN], (long)LSTRLEN(*xstr) );
return -LERR_INVAL;
}
if ( *dst != *xstr ) {
log_msg( LOG_ERROR, "bad *dst 0x%08lx need 0x%08lx", *dst, *xstr );
return -LERR_INVAL;
}
/* clean and re-init */
memset( dst, 0, xstr[LSTR_ILEN] );
*dst = *xstr;
mbr = dst+1;
buf = ((char*)dst) + LSTRLEN(*dst);
/* cvt the fixed part (occ==-1) and each occurrence of repeated part. */
for ( ;/* occ < LSTROCC(*dst) */; ) { /* cvt one part */
int i;
/* TODO: this loop is crucial for performance */
for ( i=0; i sbytes ) {
log_msg( LOG_ERROR,
"srcbuf too short %ld have %ld need %ld occ %d mbr %d",
xstr[LSTR_XLEN], sbytes, len, occ, i );
return -LERR_INVAL;
}
if ( !NEEDSWAP( xmbr ) )
/* for ( j = len; j--; ) num.buf[j] = s[j]; */
switch ( ld ) {
case 3: num.x8 = *(b8*)s; break;
case 2: num.x4 = *(b4*)s; break;
case 1: num.x2 = *(b2*)s; break;
case 0: num.buf[0] = *s; break;
}
else /* swap bytes */
for ( j = len; j--; )
num.buf[j] = s[len - 1 - j];
switch ( len ) {
case 8: *mbr = num.ll; break;
/* TODO: defines for 16 and 64 bit compilers */
case 4: *mbr = num.i; break;
case 2: *mbr = num.s; break;
case 1: *mbr = num.buf[0]; break;
}
if ( LMBRISBITS( xmbr ) ) { /* apply bit shift and mask */
*mbr >>= LMBRBITOFF( xmbr );
*mbr &= ~(-1L << LMBRBITLEN( xmbr ));
}
continue;
}
/* else raw data -- that's easy :) */
{
long offset = buf - (char*)dst;
long need;
len = LONG2LEN(xmbr);
need = offset + len;
if ( need > xstr[LSTR_ILEN] ) {
log_msg( LOG_ERROR, "bad buflen %ld need %ld+%d in occ %d mbr %d",
xstr[LSTR_ILEN], offset, len, occ, i );
return -LERR_INVAL;
}
if ( len > sbytes ) {
log_msg( LOG_ERROR,
"srcbuf too short %ld have %ld need %ld occ %d mbr %d",
xstr[LSTR_XLEN], sbytes, len, occ, i );
return -LERR_INVAL;
}
memcpy( buf, s, len );
*mbr = buf - (char*)dst;
buf += len;
}
} /* for mbrs */
if ( ++occ >= LSTROCC(*dst) )
break;
if ( occ )
part += (short)xstr[LSTR_XRLO]; /* adv. rep. part len */
else { /* was the fixed part, setup for repeated */
nmbrs = LSTRREP(*xstr);
part += (short)(xstr[LSTR_XRLO]>>16); /* adv. rep. part off */
xmbrs += i;
}
}
return 0;
} /* convert */
static int readrec ( long *dst, int fid, long where, long *xstr )
{
char *buf = (char *)
#ifdef alloca
alloca( xstr[LSTR_XLEN] )
#else
lmalloc( xstr[LSTR_XLEN] )
#endif
;
int got = 0;
int ret = 0;
if ( NULL == buf )
return -LERR_NOMEM;
if ( 0 > where )
where = -where * xstr[LSTR_XLEN];
got = readblk( buf, xstr[LSTR_XLEN], fid, where );
ret = got ? got : convert( dst, buf, xstr );
#ifndef alloca
free( buf );
#endif
return ret;
} /* readrec */
/* read and log */
static int readlog ( long *dst, int fid, long where, Db *db, int set, int rec )
{
int ret = readrec( dst, fid, where, DB_XSTR( db, set, rec ) );
if ( !ret && LOG_DO( LOG_TRACE ) )
LOG_REC( dst, lstrlib[ db->set[set] ].name[ rec ] );
return ret;
} /* readlog */
/* ************************************************************
access functions for the record types
*/
static long getOff ( Db *db, long rowid, long xr )
{
long rowix = rowid - 1; /* mfns count from 1 */
long xrf_block = rowix / 127;
long blkix = xrf_block + 1; /* ... so do xrf blocks */
long off;
if ( xrf_block < db->mmlen ) {
if ( xr )
db->mmap[xrf_block][1+(int)(rowix % 127)] = xr;
else
xr = db->mmap[xrf_block][1+(int)(rowix % 127)];
} else {
int err = 0;
if ( LOCK() ) return -3;
if ( xr ) { /* write */
if ( blkix <= db->xrlen ) {
if ( 4 != lio_pwrite( db->mst[MST_XRF], &xr, 4,
xrf_block*512 + 4*(1 + (rowix % 127))) )
return 0;
} else { /* extent */
lblk extend;
memset( extend, 0, sizeof(extend) );
while ( db->xrlen < blkix ) { /* extend */
extend[0] = db->xrlen + 1; /* set blk id */
if ( blkix == extend[0] )
extend[1+(int)(rowix % 127)] = xr;
if ( 512 != lio_pwrite( db->mst[MST_XRF], extend, 512,
db->xrlen*512) )
return 0;
db->xrlen++;
}
}
/* and go on read it back, just to check ... */
}
if ( blkix != db->xrf[LXRF_XPOS] ) {
int ret;
LOG_DBG( LOG_VERBOSE, "fetching xrf block %ld had %ld",
blkix, NULL == db->xrf ? -1 : db->xrf[LXRF_XPOS] );
ret = readlog( db->xrf, db->mst[MST_XRF],
-xrf_block, db, LSET_MST, LSTR_XRF );
if ( ret ) {
log_msg( LOG_ERROR, "\twhen fetching xrf block %ld", blkix );
err = -1;
} else if ( blkix == -db->xrf[LXRF_XPOS] ) {
LOG_DBG( LOG_DEBUG, "hmmm ... negative" );
db->xrf[LXRF_XPOS] = blkix;
} else if ( blkix != db->xrf[LXRF_XPOS] ) {
log_msg( LOG_WARN, "bad xrf %ld wanted %ld",
db->xrf[LXRF_XPOS], blkix );
err = -2;
}
}
xr = db->xrf[ LXRF_XREC + (int)(rowix % 127) ];
UNLOCK();
if ( err )
return err;
}
/*
21 bits (<<11) signed for the (512 byte) block ("xrmfb")
1 for the first block (offset 0)
0 means, never had such a record
-1 and xrmfp=0: record removed from MST
(there is no record at pos 0 in 1st block,
since there resides the MST header)
other negative value -x or pos!=0:
record logically deleted, was at +x
1 bit (1<<10): this record is new and not yet inverted
1 bit (1<<9): this record is changed and not yet re-inverted
9 bits for the block-relative position ("xrmfp")
*/
off = (((xr & 0xfffff800) >> 2) - 0x200) | (0x1ff & xr);
if ( 0 < (xr & ~0x600) ) {
LOG_DBG( LOG_DEBUG,
"offset for rowid %ld is %ld (blk %ld pos %ld) flg 0x%08x at %ld[%d]",
rowid, off, (xr>>11)&0xfffff, xr&0x1ff, xr&0x80000600, blkix, rowix%127 );
return off;
}
/* deleted */
log_msg( LOG_INFO,
"offset for rowid %ld is %ld (blk %ld pos %ld) flg 0x%08x at %ld[%d]",
rowid, off, (xr>>11)&0xfffff, xr&0x1ff, xr&0x80000600, blkix, rowix%127 );
return 0;
} /* getOff */
static long* getMfr ( Db *db, long off, long *nxtoff )
{
struct mfcxstr {
long xstr[LSTR_LONGS(7+3)];
} my = *(struct mfcxstr *)DB_XSTR( db, LSET_MST, LSTR_MFR );
long head[1+7+3];
long len = 0, base;
long *rec = NULL;
char *buf = NULL;
#ifdef alloca
int notalloca = 0;
#endif
LOG_DBG( LOG_VERBOSE, "getting MFR at off %ld", off );
if ( 498 < off % 512 )
log_msg( LOG_WARN, "blk pos > 498 in offset 0x%08x", off );
*head = *my.xstr;
if ( readrec( head, db->mst[MST_MST], off, my.xstr ) ) {
log_msg( LOG_ERROR, "\twhen reading MFR head at %ld", off );
return NULL;
}
/* log_rec( LOG_VERBOSE, head, lstrlib[LSET_MST].name[LSTR_MFR] ); */
len = head[LMFR_RECL];
LOG_DBG( LOG_VERBOSE, "got MFR %ld reclen %ld", head[LMFR_MFN], len );
if ( NULL != nxtoff ) {
*nxtoff = off + (0 < len ? len : -len); /* an odditiy */
if ( 1 & *nxtoff ) /* an odditiy */
(*nxtoff)++; /* round up to even */
if ( 498 < *nxtoff % 512 ) {
*nxtoff += 512;
*nxtoff &= ~0x1ff;
}
if ( *nxtoff > db->mfc[LMFC_NMFB]*512 + db->mfc[LMFC_NMFP] ) {
LOG_DBG( LOG_VERBOSE, "at end of db: %ld > %ld*512+%hd",
*nxtoff, db->mfc[LMFC_NMFB], db->mfc[LMFC_NMFP] );
*nxtoff = -1;
}
}
if ( len < 0 ) {
log_msg( LOG_INFO, "found deleted rec len %hd at offset %ld", len, off );
return NULL;
}
/* check external base length */
base = LONG2OFF(my.xstr[LSTR_XRLO])
+ head[LMFR_NVF]*LONG2LEN(my.xstr[LSTR_XRLO]);
if ( 0 > head[LMFR_BASE] || 0 > head[LMFR_NVF]
|| 0x8fff < head[LMFR_NVF]
|| len < head[LMFR_BASE] || len < base
|| (head[LMFR_NVF] && head[LMFR_BASE] < base)
) {
log_msg( LOG_ERROR,
"bad len %ld base %ld nvf %ld need base %ld at offset %ld",
len, head[LMFR_BASE], head[LMFR_NVF], base, off );
/* check alignment problem */
base = LONG2OFF(my.xstr[LSTR_XRLO])
+ head[LMFR_STAT]*LONG2LEN(my.xstr[LSTR_XRLO]);
if ( 0 > head[LMFR_NVF] || 0 > head[LMFR_STAT]
|| 0x8fff < head[LMFR_STAT]
|| len < head[LMFR_NVF] || len < base
|| (head[LMFR_STAT] && head[LMFR_NVF] < base)
)
;
else
log_msg( LOG_ERROR, "probably alignment problem, try -format aligned" );
goto cleanup;
}
/* set up external structure for this rec */
my.xstr[LSTR_SIZE] |= head[LMFR_NVF] << 16; /* occ of rep. part */
my.xstr[LSTR_XLEN] = len;
/* internal base length */
base = LSTRLEN( *my.xstr );
/* internal len adjusted for the slightly longer base */
my.xstr[LSTR_ILEN] = len + base - head[LMFR_BASE]; /* the buffer */
rec = nrec( my.xstr );
if ( NULL == rec ) {
log_msg( LOG_SYSERR, "could not alloc MFR of len %hd", my.xstr[LSTR_ILEN] );
goto cleanup;
}
buf = (char *)
#ifdef alloca
alloca( len );
/* first try faster alloca, but stack may be too limited for large records */
notalloca = NULL == buf;
if ( notalloca )
buf =
#endif
lmalloc( len );
if ( NULL == buf ) {
log_msg( LOG_SYSERR, "could not alloc MFR of len %hd", len );
goto cleanup;
}
if ( readblk( buf, len, db->mst[MST_MST], off ) ) {
log_msg( LOG_ERROR, "\twhen reading MFR" );
goto cleanup;
}
if ( LVAR_PAC != (DB_VARI & db->flags) ) {
if ( convert( rec, buf, my.xstr ) ) {
log_msg( LOG_ERROR, "\twhen converting MFR" );
goto cleanup;
}
} else { /* 10% faster */
Mfr *mfr = (Mfr*)buf;
short *s = &mfr->dict->tag;
long *f = rec + LMFR__FL;
long *e = f + 3*head[LMFR_NVF];
rec[LMFR_MFN] = mfr->mfn;
rec[LMFR_RECL] = mfr->recl;
/*
rec[LMFR_BWB] = mfr->bwbh<<16 | mfr->bwbl;
rec[LMFR_BWP] = mfr->bwp;
*/
rec[LMFR_BASE] = mfr->base;
rec[LMFR_NVF] = mfr->nvf;
rec[LMFR_STAT] = mfr->stat;
while ( f < e ) {
*f++ = *s++;
*f++ = *s++;
*f++ = *s++;
}
}
if ( rec[LMFR_STAT] ) {
log_msg( LOG_WARN, "found status %hd", rec[LMFR_STAT] );
goto cleanok;
}
/* do a consistency check */
if ( rec[LMFR_NVF] < 0 || rec[LMFR_BASE] < 0 ) {
log_msg( LOG_ERROR, "found neg. field nvf %hd base %hd",
rec[LMFR_NVF], rec[LMFR_BASE] );
goto cleanup;
}
/* now care for the field values */
{
char *valsrc = buf+rec[LMFR_BASE];
char *recsta = ((char*)rec);
char *valdst = recsta + base;
long xbufl = rec[LMFR_RECL] - rec[LMFR_BASE];
long sumlens = 0;
int i;
for ( i=0; i < rec[LMFR_NVF]; i++ ) {
long *d = &rec[LMFR__FL + i*LMFR__RL];
if ( d[LMFR_POS] < 0 || d[LMFR_LEN] < 0 ) {
log_msg( LOG_ERROR,
"bad field %d at off %ld: negativ pos %hd or len %hd",
i, off, d[LMFR_POS], d[LMFR_LEN] );
goto cleanup;
}
if ( d[LMFR_POS] + d[LMFR_LEN] > xbufl ) {
log_msg( LOG_ERROR,
"bad field %d at off %ld: pos %hd + len %hd > buf %ld",
i, off, d[LMFR_POS], d[LMFR_LEN], xbufl );
goto cleanup;
}
sumlens += d[LMFR_LEN];
if ( sumlens > xbufl ) {
log_msg( LOG_ERROR,
"bad fields at off %ld: sum of lengths %ld > buf %ld",
off, sumlens, xbufl );
goto cleanup;
}
memcpy( valdst, valsrc+d[LMFR_POS], d[LMFR_LEN] );
d[LMFR_POS] = valdst - recsta;
valdst += d[LMFR_LEN];
}
} /* consistency check */
rec[LMFR_BWB] = /* "used" bytes */
rec[LMFR_RECL] = my.xstr[LSTR_ILEN];
rec[LMFR_BWP] = rec[LMFR_NVF]; /* avail fields = used fields */
rec[LMFR_BASE] = base;
if ( LOG_TRACE <= log_lev )
LOG_REC( rec, lstrlib[LSET_MST].name[LSTR_MFR] );
goto done;
cleanup:
if ( NULL != nxtoff )
*nxtoff = -1;
cleanok:
if ( NULL != rec ) {
free( rec );
rec = NULL;
}
done:
if ( NULL != buf
#ifdef alloca
&& notalloca
#endif
)
free( buf );
if ( NULL != rec )
*rec = db->dbid;
return rec;
} /* getMfr */
/** write the record.
If it doesn't yet have a mfn, assign one.
*/
static int putMfr ( Db *db, Mfr *mfr )
{
long oldpos, newpos;
int ret;
if ( !(db->flags & DB_WRITABLE) ) {
log_msg( LOG_ERROR, "db is not writable" );
return -1;
}
/* minimalist sanity check */
if ( mfr->nvf < 0
|| mfr->base != 18 + 6*mfr->nvf
|| mfr->base > mfr->recl
) {
log_msg( LOG_ERROR, "bad nvf/base/recl %d/%d/%d ",
mfr->nvf, mfr->base, mfr->recl );
return -2;
}
db->flags |= DB_MODIFIED;
if ( mfr->mfn ) {
long block;
oldpos = getOff( db, mfr->mfn, 0 );
mfr->bwp = 511 & oldpos;
block = 1 + (oldpos >> 9); /* blockno counting from 1 */
mfr->bwbl = 0xffff & block;
mfr->bwbh = block >> 16;
if ( db->mfc[LMFC_NMFN] <= mfr->mfn )
db->mfc[LMFC_NMFN] = mfr->mfn+1;
} else {
mfr->mfn = db->mfc[LMFC_NMFN]++; /* assign new mfn */
oldpos = 0;
mfr->bwbl = mfr->bwbh = mfr->bwp = 0;
}
mfr->stat = 0;
newpos = db->mflen;
if ( 498 < (newpos & 511) ) /* round up to next block boundary */
newpos = ~511 & (newpos + 14);
if ( 1 & newpos )
newpos++;
db->mflen = newpos + mfr->recl;
ret = lio_pwrite( db->mst[MST_MST], (char*)mfr, mfr->recl, newpos );
if ( ret != mfr->recl )
return log_msg( -LERR_TRASH, "could not write Mfr %d bytes got %d",
mfr->recl, ret );
getOff( db, mfr->mfn, (1 << (oldpos ? 9 : 10))
| (((newpos & 0xfffffe00) + 0x200) << 2) | (0x1ff & newpos) );
return 0;
} /* putMfr */
static int putRec ( Db *db, OpenIsisRec *rec )
{
int ret = 0, i;
Mfr *mfr = NULL;
int buflen = 0;
int reclen = 0;
int contig = 1;
#ifdef alloca
int notalloca = 0;
#endif
const char *rbase = ((char *)rec) + rec->base;
Dict *d;
OpenIsisField *f = rec->field;
for ( i = rec->len; i--; f++ ) {
if ( ! f->len )
continue;
if ( NULL == f->val ) {
log_msg( LOG_ERROR, "bad rec NULL val" );
ret = -LERR_FAULT;
goto cleanup;
}
contig = contig && (f->val == rbase + buflen);
buflen += f->len;
}
reclen = 18 + 6*rec->len + buflen;
if ( 1 & reclen )
reclen++;
mfr = (Mfr*)
#ifdef alloca
alloca( reclen );
notalloca = NULL == mfr;
if ( notalloca )
mfr = (Mfr*)
#endif
lmalloc( reclen );
if ( NULL == mfr ) {
log_msg( LOG_SYSERR, "could not alloc MFR of len %hd", reclen );
ret = -LERR_NOMEM;
goto cleanup;
}
mfr->mfn = rec->rowid;
mfr->recl = reclen;
mfr->bwbl = mfr->bwbh = mfr->bwp = 0;
mfr->base = 18 + 6*rec->len;
mfr->nvf = rec->len;
mfr->stat = 0;
d = mfr->dict;
f = rec->field;
buflen = 0;
for ( i = rec->len; i--; d++, f++ ) {
d->tag = f->tag;
d->pos = buflen;
buflen += (d->len = f->len);
}
if ( contig )
memcpy( ((char*)mfr)+mfr->base, rbase, buflen );
else {
char *mbase = ((char*)mfr)+mfr->base;
d = mfr->dict;
f = rec->field;
for ( i = rec->len; i--; d++, f++ )
if ( d->len )
memcpy( mbase + d->pos, f->val, d->len );
}
ret = putMfr( db, mfr );
if ( !ret && !rec->rowid )
rec->rowid = mfr->mfn;
cleanup:
if ( NULL != mfr
#ifdef alloca
&& notalloca
#endif
)
free( mfr );
return ret;
} /* putRec */
/* ************************************************************
utilities
*/
static int argchk ( const char *param, const char *n, const char *v )
{
return strcmp( param, n ) ? 0 : NULL != v ? 1 :
(log_msg( LOG_ERROR, "no value for param '%s'", param ), 0);
} /* argchk */
/* ************************************************************
package data
*/
/* ************************************************************
package functions
*/
long *ldb_readRecAtOff ( int dbid, lxref off, long *nxtoff )
{
long *rec;
Db *db = getDb( dbid );
if ( NULL == db ) {
log_msg( LOG_ERROR, "\tat ldb_readRecAtOff" );
return NULL;
}
if ( 0 == off )
off = 64;
rec = getMfr( db, off, nxtoff );
if ( NULL == rec )
return NULL;
LOG_DBG( LOG_VERBOSE, "db %d off %ld: got %hd bytes",
dbid, off, NULL==rec ? -1 : rec[LMFR_RECL] );
return rec;
} /* ldb_readRecAtOff */
OpenIsisRaw *openIsisReadRaw ( int dbid, long rowid )
{
long off;
long *rec;
Db *db = getDb( dbid );
if ( NULL == db ) {
log_msg( LOG_ERROR, "\tat openIsisReadRaw %ld", rowid );
return NULL;
}
off = getOff( db, rowid, 0 );
log_msg( LOG_INFO, "found xref 0x%08lx for %ld", off, rowid );
if ( 0 >= off ) {
log_msg( LOG_INFO, "found deleted xref 0x%08lx for %ld", off, rowid );
return NULL;
}
rec = getMfr( db, off, NULL );
if ( NULL == rec ) {
log_msg( LOG_WARN, "\tno record at %ld rowid %ld", off, rowid );
return NULL;
}
LOG_DBG( LOG_VERBOSE, "db %d row %ld: got %hd bytes",
dbid, rowid, NULL==rec ? -1 : rec[LMFR_RECL] );
if ( rec[LMFR_MFN] != rowid ) {
log_msg( LOG_ERROR, "got mfn %ld expected %ld", rec[LMFR_MFN], rowid );
free( rec );
return NULL;
}
return (OpenIsisRaw*)rec;
} /* openIsisReadRaw */
int openIsisWrite ( int dbid, OpenIsisRec *rec )
{
Db *db = getDb( dbid );
if ( NULL == db )
return -LERR_BADF;
return putRec( db, rec );
} /* openIsisWrite */
int ldb_search ( int dbid, const char *key, LdbPost *post,
OpenIsisRec *rec )
{
int i, j, prefix, idx, ret, ock;
long pos;
long *leaf, *entry;
char *term;
long *xstr;
struct { /* terms cursor */
char key[LDB_MAX_KEYLEN+1]; /* key or key prefix */
short klen; /* key length to compare */
char imin; /* minimum index to search */
char imax; /* maximum index to search */
long leaf[LDB_INDEXES][LDB_TERMBUF]; /* one leaf buffer per index */
short lpos[LDB_INDEXES]; /* next position in leaf, -1 if done */
} crs;
short klen; /* length for initial locate */
long block[128]; /* buffer to read one block */
long blockpos = 0;
LdbP *p = NULL;
Db *db = getDb( dbid );
if ( NULL == db )
return -LERR_BADF;
if ( NULL == key )
key = "$";
/* prepare cursor struct */
memset( &crs, 0, sizeof(crs) ); /* tabula rasa */
crs.klen = strlen( key );
/* check for prefix match */
if ( NULL != post )
prefix = LDB_PFX & post->mode;
else if ( (prefix = crs.klen && '$' == key[crs.klen - 1]) )
crs.klen--;
/* check out minimum index to search */
for ( crs.imin=0; crs.klen > db->tlen[(int)crs.imin]; )
if ( LDB_INDEXES == ++(crs.imin) ) {
log_msg( LOG_ERROR, "bad keylen %d key '%.64s'", crs.klen, key );
return -LERR_INVAL;
}
/* prepare key */
memset( crs.key, ' ', sizeof(crs.key)-1 );
{
unsigned char *uc = (unsigned char*)crs.key;
unsigned char *uk = (unsigned char*)key;
for ( i=crs.klen; i--; )
uc[i] = db->ctab[LDB_UCTAB][ uk[i] ];
}
if ( prefix )
crs.imax = LDB_INDEXES-1;
else {
crs.imax = crs.imin;
crs.klen = db->tlen[(int)crs.imin];
}
log_msg( LOG_INFO, "search for '%.*s'%c", crs.klen, crs.key, prefix?'$':' ' );
key = crs.key;
klen = crs.klen;
if ( NULL != rec && rec->len ) {
/* use last key from record to locate starting position */
key = rec->field[rec->len-1].val;
klen = rec->field[rec->len-1].len;
rec->len = 0;
}
for ( i=crs.imin; i<=crs.imax; i++ ) { /* find leaf positions */
int nFile = INV_N01 + 2*i; /* node file index */
int nStr = LSTR_N01 + 2*i; /* node struct index */
long *nstr = DB_XSTR( db, LSET_INV, nStr );
int lvl;
short cmplen = klen <= db->tlen[i] ? klen : db->tlen[i];
pos = db->cnt[i][LCNT_POSR]; /* pos of root record */
j = 0;
for ( lvl = 0; 0= nstr[LSTR_ILEN] );
*node = *nstr;
ret = readlog( node, db->inv[nFile], 1-pos, db, LSET_INV, nStr );
if ( pos != node[LN0X_POS] /* wrong address */
|| i+1 != node[LN0X_TYPE] /* wrong type */
|| node[LN0X_OCK] < 1 /* no keys */
|| 2*db->cnt[i][LCNT_ORDN] < node[LN0X_OCK] /* too many keys */
) {
log_msg( LOG_ERROR, "bad node pos %ld type %ld keys %ld",
node[LN0X_POS], node[LN0X_TYPE], node[LN0X_OCK]
);
return -LERR_TRASH;
}
ock = node[LN0X_OCK];
for ( j=1;
jlen )
post->len = sizeof(post->p)/sizeof(post->p[0]); /* standard length */
if ( LDB_NOT & post->mode )
post->mode |= LDB_AND;
p = post->p;
}
xstr = DB_XSTR( db, LSET_INV, LSTR_IFP );
for (;;) { /* loop terms in prefix mode */
/* vars for postings: */
long infb, infp; /* block and pos where to read postings */
int added; /* postings added or marked per term */
int blkno; /* postings block number */
int remain = 0; /* postings to fetch from next block of segment */
long ifp[LIFP__FL]; /* postings header */
idx = -1; /* index to use */
/* compare index terms, load leafes if needed */
for ( i = crs.imin; i <= crs.imax; i++ ) {
short cmplen = klen <= db->tlen[i] ? klen : db->tlen[i];
leaf = crs.leaf[i];
if ( leaf[LL0X_OCK] <= crs.lpos[i] ) { /* load */
int lFile = INV_L01 + 2*i; /* leaf file index */
int lStr = LSTR_L01 + 2*i; /* leaf struct index */
long *lstr = DB_XSTR( db, LSET_INV, lStr );
crs.lpos[i] = -1;
reread:
if ( ! (pos = leaf[LL0X_PS]) )
continue;
LOG_DBG( LOG_DEBUG, "leaf %ld", pos );
assert( (long)sizeof(crs.leaf[i]) >= lstr[LSTR_ILEN] );
*leaf = *lstr;
ret = readlog( leaf, db->inv[lFile], 1-pos, db, LSET_INV, lStr );
if ( pos != leaf[LL0X_POS] /* wrong address */
|| i+1 != leaf[LL0X_TYPE] /* wrong type */
|| leaf[LL0X_OCK] < 1 /* no keys */
|| 2*db->cnt[i][LCNT_ORDN] < leaf[LL0X_OCK] /* too many keys */
) {
log_msg( LOG_ERROR, "bad leaf pos %ld type %ld keys %ld",
leaf[LL0X_POS], leaf[LL0X_TYPE], leaf[LL0X_OCK] );
return -LERR_TRASH;
}
ock = leaf[LL0X_OCK];
/* advance to first term which is not too small
(should be needed only for first leaf of an index)
*/
for ( j=0;
j crs.lpos[i] )
continue;
if ( 0 > idx ) {
idx = i;
continue;
}
/* compare this index next term to that of index idx */
/* assume that index w/ lower number has shorter keys */
ret = memcmp(
((char*)leaf)+leaf[LL0X__FL + crs.lpos[i]*LL0X__RL + LL0X_KEY],
((char*)crs.leaf[idx])+
crs.leaf[idx][LL0X__FL + crs.lpos[idx]*LL0X__RL + LL0X_KEY],
db->tlen[idx] );
if ( 0 > ret )
idx = i;
}
if ( 0 > idx )
goto done;
j = crs.lpos[idx];
leaf = crs.leaf[idx];
entry = leaf + LL0X__FL + j*LL0X__RL;
term = ((char*)leaf) + entry[LL0X_KEY];
if ( memcmp( crs.key, term, crs.klen ) )
goto done;
crs.lpos[idx]++;
if ( NULL != rec ) { /* record the term */
/* field to assign */
OpenIsisField *f = rec->field + rec->len;
short tlen = db->tlen[idx];
/* end of available buffer */
char *b = rec->len
? (char*)f[-1].val /* before previously assigned field */
: ((char*)rec + rec->bytes); /* end of record */
while ( tlen && ' ' == term[tlen-1] )
tlen--;
b -= tlen;
if ( b < (char*)(f+1) )
goto done;
/* probably we're nuking the locator now: */
memcpy( b, term, tlen );
f->tag = 0;
f->val = b;
f->len = tlen;
rec->len++;
/* reset key from locator to prefix */
key = crs.key;
klen = crs.klen;
}
if ( NULL == post )
continue;
/* collect postings */
infb = entry[LL0X_INFB];
infp = entry[LL0X_INFP];
/* the IFP file is organized in blocks of 128 longs.
1st long is block number followed by 127 data.
postings are organized in chained segments so that each segment
fits within one such block. a segment has five longs header,
giving number of postings and pointer to next segment.
*/
added = 0;
for ( blkno=0; infb; blkno++ ) { /* segments */
LdbP merge[127/2]; /* buffer to collect new postings */
long *base; /* start of data */
long *b; /* start of postings */
long n; /* max postings in this seg's 1st block */
long xlen; /* external length to read */
int f = post->fil - 1; /* highest pos to consider in given postings */
int m = 0; /* fill merge buffer */
int k; /* loop segment */
if ( infp > 127-2-5 ) {
log_msg( LOG_ERROR, "found bad IFP pos %ld blk %d for %.*s",
infp, blkno, klen, key );
return -LERR_TRASH;
}
if ( remain ) { /* consecutive block of same segment */
n = remain;
if ( n > 127/2 )
n = 127/2;
xlen = 8*n;
} else {
n = (127 - 5 - infp)/2;
xlen = 20 + 8*n;
}
pos = (infb - 1) * 512 + (infp + 1) * 4;
if ( blockpos
&& !((pos-blockpos) >> 9) /* 0 <= (pos-blockpos) < 512 */
&& pos+xlen <= blockpos+ 1 + (0x1ff & ~blockpos)
)
base = block + (pos - blockpos)/sizeof(long);
else {
long blklen = 1 + (0x1ff & ~pos);
assert( xlen <= blklen );
assert( blklen <= (long)sizeof(block) );
assert( 0 == (0x1ff & (pos + blklen)) );
ret = readblk( block, blklen, db->inv[INV_IFP], pos );
if ( ret ) {
log_msg( LOG_ERROR, "\twhen reading IFP" );
return -LERR_IO;
}
blockpos = pos;
base = block;
}
if ( remain ) { /* no header to convert */
remain -= n;
b = base; /* no header */
} else {
assert( (long)sizeof(ifp) >= xstr[LSTR_ILEN] );
*ifp = *xstr;
ret = convert( ifp, (char *)base, xstr );
if ( ret ) {
log_msg( LOG_ERROR, "\twhen converting IFP header" );
return -LERR_TRASH;
}
if ( n > ifp[LIFP_SEGP] )
n = ifp[LIFP_SEGP];
remain = ifp[LIFP_SEGP] - n;
b = base+5; /* after header */
}
LOG_DBG( LOG_VERBOSE,
"key %ld.%d '%.*s' blk %d post %d/%d r %d xlen %ld at b/p %ld.%ld=%ld",
leaf[LL0X_PS], j, db->tlen[idx], term, blkno,
n, ifp[LIFP_TOTP], remain, xlen, infb, infp, pos );
if ( LOG_DO( LOG_TRACE ) )
LOG_REC( ifp, lstrlib[ db->set[LSET_INV] ].name[ LSTR_IFP ] );
assert( (size_t)n <= sizeof(merge)/sizeof(merge[0]) );
for ( k=n; k--; ) {
long prow, ptag, ppos;
unsigned char *c = (unsigned char *)&b[k*2];
LdbP e; /* the entry */
LdbP samerow; /* highest possible entry w/ same row as e */
#if #cpu (sparc)
/* the 8 bytes of a posting are BIG ENDIAN ! */
memcpy(e.bytes,c,8);
#else
e.bytes[0] = c[7]; e.bytes[1] = c[6];
e.bytes[2] = c[5]; e.bytes[3] = c[4];
e.bytes[4] = c[3]; e.bytes[5] = c[2];
e.bytes[6] = c[1]; e.bytes[7] = c[0];
#endif
prow = LDBP_ROW( &e );
ptag = LDBP_TAG( &e );
ppos = LDBP_POS( &e );
LDBP_SETROWTOP( &samerow, &e ); /* for mfn comparison */
LOG_DBG( LOG_VERBOSE, "post %ld.%hd pos %06lx key '%.*s'",
prow, ptag, ppos, db->tlen[idx], term );
if ( prow < post->skp ) /* quickly bail out on skip condition */
break;
if ( 0 >= ptag /* bad tag */
|| !prow || prow >= db->mfc[LMFC_NMFN] /* bad mfn */
|| (post->cut && prow >= post->cut)
|| (post->tag && post->tag != ptag)
)
continue;
#ifdef TRY_BINSEARCH_SPEEDUP
if ( post->fil < 8*n ) {
#endif
/* simple sweep ... */
while ( f >= 0 && LDBP_GT( p+f, &samerow ) )
f--;
#ifdef TRY_BINSEARCH_SPEEDUP
} else {
assert( NULL == "binsearch not implemented yet" );
}
#endif
if ( LDB_AND & post->mode ) {
int l;
/* loop postings for same row, mark all (that are near enough) */
LDBP_SETROWBOT( &samerow, &e ); /* for mfn comparison */
/* NOTE: postings for row are GT than bottom even if marked */
for ( l = f; l>=0 && LDBP_GT( p+l, &samerow ); l-- ) {
if ( post->near ) {
long dist;
if ( ptag != LDBP_TAG( p+l ) ) continue;
if ( LDB_NEAR_G != post->near ) {
dist = LDBP_POS( p+l ) - LDBP_POS( &e );
if ( dist < 0 ) dist = -dist;
if ( 0 < post->near
? post->near < dist
: -post->near != dist /* exact $$$$ */
) continue;
}
}
LDBP_SETMARK( p+l );
added++;
}
} else { /* OR mode */
int add;
if ( ! post->near ) /* add if row not found: ignore details */
add = 0 > f || prow > LDBP_ROW( p+f );
else { /* add if no exact match */
int l;
/* NOTE: we don't use mark bit in OR mode, do we ? */
for ( l = f; l>=0 && LDBP_GT( p+l, &e ); l-- )
;
add = 0 > l || LDBP_GT( &e, p+l );
}
if ( add )
merge[ m++ ] = e;
}
} /* for postings in segment */
if ( m ) { /* merge in the merge buffer */
LdbP *mm = merge;
added += m;
for ( k = post->fil += m; m && k--; ) {
LdbP src;
if ( k < m || LDBP_GT( mm, &p[k-m] ) ) {
src = *mm++;
m--;
LOG_DBG( LOG_DEBUG, "merging %ld at %d", LDBP_ROW(&src), k );
} else
src = p[k-m];
if ( k < post->len )
p[k] = src;
else { /* set cut */
long row = LDBP_ROW( &src );
if ( row < post->cut || !post->cut )
post->cut = row;
}
}
if ( post->fil > post->len )
post->fil = post->len;
if ( post->cut ) /* postings for cut row are unreliable */
while ( post->fil && post->cut <= LDBP_ROW(p+post->fil-1) )
post->fil--;
}
if ( remain ) { /* advance to start of next block */
infb++;
infp = 0;
} else {
infb = ifp[LIFP_NXTB];
infp = ifp[LIFP_NXTP];
}
} /* for segments */
LOG_DBG( LOG_VERBOSE, "added %d postings for key '%.*s'",
added, db->tlen[idx], term );
} /* for terms in prefix/postings mode */
done:
if ( NULL != post /* fixup */
&& LDB_AND & post->mode && !(LDB_KEEPMARKS & post->mode)
) {
int mark = LDB_NOT & post->mode ? 0 : 0x8000;
j=0;
for ( i=0; ifil; i++ )
if ( mark == LDBP_MARK(p+i) ) {
LDBP_CLRMARK(p+i);
p[j++] = p[i];
}
post->fil = j;
}
return NULL == rec ? 0 : rec->len;
} /* ldb_search */
long ldb_p2s ( OpenIsisSet *set, LdbPost *post )
{
long *s = set->id;
long last=0, max = set->len;
int i;
set->len = 0;
if ( ! max )
max = OPENISIS_SETLEN;
max--;
if ( !post->fil )
return 0L;
s[0] = LDBP_ROW(post->p);
for ( i=1; ifil && last < max; i++ ) {
long row = LDBP_ROW(post->p+i);
if ( s[last] != row )
s[++last] = row;
}
return set->len = last+1;
} /* ldb_p2s */
LdbCTab *ldb_tabs( int dbid )
{
Db *db = getDb( dbid );
return NULL == db ? NULL : db->ctab;
} /* ldb_tabs */
/* ************************************************************
public functions
*/
int openIsisInit ( int argc, const char **argv,
OpenIsisLockFunc lockfunc )
{
(void)argc; (void)argv;
lock = lockfunc;
return 0;
}
int openIsisOpen ( const char *dbname, const char **argv, int argc )
{
static int init;
Db ndb, *db;
int dbid, ret = 0, internaldump = 0, reopen = 0, invret = 0;
int autoformat = 1; /* guess alignment */
int writable = 1;
const char *actab = NULL, *uctab = NULL;
memset( &ndb, 0, sizeof(ndb) );
/* these should be NULL by memsetting to 0 anyway ... */
ndb.name = NULL; ndb.path = NULL; ndb.format = NULL; ndb.mmap = NULL;
#ifndef LACKING_MMAP
ndb.flags |= DB_MMAP; /* set unconditionally ? */
#endif
/* check options ... */
if ( NULL != dbname )
ndb.name = dbname;
while ( argc ) {
const char *n = argv[0], *v = NULL;
assert( n );
if ( '-' == n[0] )
n++;
if ( 1 == argc || '-' == argv[1][0] ) { /* no value */
argc--;
argv++;
} else {
v = argv[1];
assert( v );
argc -= 2;
argv += 2;
}
if ( !strcmp("v",n) )
openIsisLog( NULL == v ? LOG_ALL : *v, NULL );
else if ( argchk("db",n,v) )
ndb.name = v;
else if ( argchk("dbpath",n,v) )
ndb.path = v;
else if ( argchk("format",n,v) )
ndb.format = v;
else if ( argchk("actab",n,v) )
actab = v;
else if ( argchk("uctab",n,v) )
uctab = v;
else if ( !strcmp("internaldump",n) )
internaldump = !0;
else if ( !strcmp("reopen",n) )
reopen = !0;
} /* while argc */
if ( NULL == ndb.name )
return -LERR_FAULT;
/* more init AFTER honoring verbosity */
if ( ! init ) {
lstr_auto(NULL);
init = !0;
}
for ( dbid=0; dbiddbid = dbid;
/* save params */
if ( NULL != db->name ) db->name = strdup( db->name );
if ( NULL != db->path ) db->path = strdup( db->path );
if ( NULL != db->format ) db->format = strdup( db->format );
if ( NULL == db->format )
;
else if ( !strncmp( "naligned", db->format, 3 ) )
autoformat = 0;
else if ( !strncmp( "aligned", db->format, 3 ) ) {
autoformat = 0;
db->flags |= LVAR_ALI;
} else {
log_msg( LOG_ERROR, "unknown format '%s'", db->format );
}
db->set[ LSET_MST ] = LSET_MST;
db->set[ LSET_INV ] = LSET_INV;
/* preset record sizes */
db->mfc[0] = *DB_XSTR( db, LSET_MST, LSTR_MFC );
db->xrf[0] = *DB_XSTR( db, LSET_MST, LSTR_XRF );
db->cnt[0][0] =
db->cnt[1][0] = *DB_XSTR( db, LSET_INV, LSTR_CNT );
/* isis-1 index term lengths */
db->tlen[0] = 10;
db->tlen[1] = 30;
/* only the packed little endian ("DOS") format is writable */
if ( LVAR_PAC != (DB_VARI & db->flags) )
writable = 0;
/* open files */
if ( writable
&& !openfiles( db->mst, db->path, db->name, EXT_MST, MST_FILES, !0 )
)
ret = 0;
else {
writable = 0;
ret = openfiles( db->mst, db->path, db->name, EXT_MST, MST_FILES, 0 );
if ( ret )
goto cleanup;
}
if ( writable
&& !openfiles( db->inv, db->path, db->name, EXT_INV, INV_FILES, !0 )
)
invret = 0;
else {
writable = 0;
invret = openfiles( db->inv, db->path, db->name, EXT_INV, INV_FILES, 0 );
}
if ( autoformat ) {
if ( invret )
log_msg( LOG_WARN, "cannot guess format -- no inverted file" );
else {
unsigned long len = lio_size( db->inv[INV_CNT] );
if ( 56L == len ) {
db->flags |= LVAR_ALI;
writable = 0; /* we do not write aligned format */
} else if ( 52L != len )
log_msg( LOG_WARN, "cannot guess format -- bad .cnt len %ld", len );
log_msg( LOG_INFO, "using autoformat %saligned for .cnt len %ld",
(db->flags & LVAR_ALI) ? "":"un", len );
}
}
if ( (ret = readlog( db->mfc, db->mst[MST_MST], 0, db, LSET_MST, LSTR_MFC )) )
goto cleanup;
db->mflen = lio_size( db->mst[MST_MST] );
{
long lastblock = db->mflen/512;
if ( 511 & db->mflen ) lastblock++; /* counting from 1 */
/*
the next record's block should be either the last one we have
or the next one to follow
*/
if ( db->mfc[LMFC_NMFB] != lastblock && db->mfc[LMFC_NMFB] != lastblock+1 )
log_msg( LOG_WARN, "NMFB mismatch: NMFB %ld ~ %ld",
db->mfc[LMFC_NMFB], lastblock );
/* set LOGICAL mf length */
db->mflen = (db->mfc[LMFC_NMFB]-1)*512 + db->mfc[LMFC_NMFP];
}
db->xrlen = lio_size( db->mst[MST_XRF] ) / 512;
#ifndef LACKING_MMAP
if ( (DB_MMAP & db->flags) && db->xrlen ) {
if ( MAP_FAILED != (db->mmap = mmap( NULL, db->xrlen*512,
PROT_READ | (writable ? PROT_WRITE : 0),
MAP_SHARED, db->mst[MST_XRF], 0 ))
)
db->mmlen = db->xrlen;
else {
db->mmap = NULL;
db->flags &= ~DB_MMAP;
}
}
#endif
/* ctables */ /* TODO: care for actab/uctab params */
{ /* standard alphanum table */
memset( db->ctab[LDB_ACTAB]+'0', 1, 10 );
memset( db->ctab[LDB_ACTAB]+'A', 1, 26 );
memset( db->ctab[LDB_ACTAB]+'a', 1, 26 );
memset( db->ctab[LDB_ACTAB]+128, 1, 128 );
}
{ /* standard upcase/normalization table */
int i;
/* C0 controls go blank */
for ( i=0; i<=31; i++ ) db->ctab[LDB_UCTAB][i] = (unsigned char)' ';
/* default: leave alone */
for ( i=32; i<=255; i++ ) db->ctab[LDB_UCTAB][i] = (unsigned char)i;
/* upcase little ASCIIs */
for ( i='a'; i<='z'; i++ ) db->ctab[LDB_UCTAB][i] = (unsigned char)(i-32);
}
/* done */
db->flags |= DB_OPEN;
if ( writable )
db->flags |= DB_WRITABLE;
if ( internaldump ) {
long off = 0;
long *r;
do {
if ( NULL != (r = ldb_readRecAtOff(dbid,off,&off)) )
free( r );
} while ( 0 < off );
exit(0);
}
if ( invret )
goto noinv;
if ( (ret = readlog( db->cnt[0], db->inv[INV_CNT],
0, db, LSET_INV, LSTR_CNT ))
|| (ret = readlog( db->cnt[1], db->inv[INV_CNT],
-1, db, LSET_INV, LSTR_CNT ))
)
goto noinv;
/* done */
db->flags |= DB_INVOPEN;
noinv:
return dbid;
cleanup:
/* cleanup ... */
return 0 > ret ? ret : ret ? -ret : -1;
} /* openIsisOpen */
int openIsisClose ( int dbid )
{
Db *db = getDb( dbid );
if ( NULL == db )
return -LERR_BADF;
if ( LOCK() ) return -LERR_BUSY;
if ( DB_MODIFIED & db->flags ) {
/* write back the MF control */
Mfc mfc;
/* if ( 498 < (db->mflen & 511) ) db->mflen = ~511 & (db->mflen + 14); */
mfc.ctlm = db->mfc[LMFC_CTLM];
mfc.nmfn = db->mfc[LMFC_NMFN];
mfc.nmfb = 1 + (db->mflen >> 9);
mfc.nmfp = 511 & db->mflen;
mfc.type = db->mfc[LMFC_TYPE];
mfc.rcnt = db->mfc[LMFC_RCNT];
mfc.mfx1 = db->mfc[LMFC_MFX1];
mfc.mfx2 = db->mfc[LMFC_MFX2];
mfc.mfx3 = db->mfc[LMFC_MFX3];
if ( sizeof(mfc) != lio_pwrite( db->mst[MST_MST], &mfc, sizeof(mfc), 0) )
log_msg( -LERR_TRASH, "could not write MST header" );
}
#ifndef LACKING_MMAP
if ( NULL != db->mmap )
munmap( db->mmap, db->mmlen*512 );
#endif
closefiles( db->mst, MST_FILES );
if ( DB_INVOPEN & db->flags )
closefiles( db->inv, INV_FILES );
db->flags = 0L;
if ( NULL != db->name ) free( (char*)db->name );
if ( NULL != db->path ) free( (char*)db->path );
if ( NULL != db->format ) free( (char*)db->format );
memset( db, 0, sizeof(db) );
UNLOCK();
return 0;
} /* openIsisClose */
long openIsisMaxRowid ( int dbid )
{
Db *db = getDb( dbid );
if ( NULL == db )
return -LERR_BADF;
return db->mfc[LMFC_NMFN] - 1;
} /* openIsisMaxRowid */
OpenIsisRec* openIsisTerm ( OpenIsisRec *rec,
int dbid, const char *key )
{
return 0 > ldb_search( dbid, key, NULL, rec ) ? NULL : rec;
} /* openIsisTerm */
int openIsisCheck ( int dbid, int flags )
{
static char dot = '.';
long *r;
Db *db = getDb( dbid );
long nxtoff = 64, off;
if ( NULL == db )
return -LERR_BADF;
(void)flags;
do {
lio_write( 0, &dot, 1 );
if ( NULL != (r = ldb_readRecAtOff(dbid,off=nxtoff,&nxtoff)) ) {
long o = getOff( db, r[LMFR_MFN], 0 );
if ( o != off ) {
log_msg( LOG_WARN, "mfn %ld xrf %ld != real %ld\n",
r[LMFR_MFN], o, off );
}
free( r );
}
} while ( 0 < nxtoff );
return 0;
} /* openIsisCheck */
./ldb.h 0100644 0001130 0001130 00000013570 07447676140 010627 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
#ifndef LDB_H
/*
$Id: ldb.h,v 1.12 2002/03/25 19:36:44 kripke Exp $
package interface of the general db access functions.
*/
#include "luti.h"
/* #define LDB_MAGIC 0x0FE91515 read OPENISIS */
enum { /* MFR members */
LMFR_MFN = 1, /* the rowid */
LMFR_RECL, /* total external length (even), negative for locked record */
LMFR_BWB, /* block of rec's previous version */
LMFR_BWP, /* pos of " */
LMFR_BASE, /* offset of contents area */
LMFR_NVF, /* number of fields */
LMFR_STAT, /* state, if != 0, about to be deleted */
LMFR__FL, /* offset of repeated part */
LMFR_TAG = 0,
LMFR_POS,
LMFR_LEN,
LMFR__RL /* length of repeated part */
};
/**
read a raw isis record by rowid.
the memory must be freed after usage.
*/
extern long *ldb_readRec ( int db, long rowid );
/**
numerical type for a records file position.
To support large DBs, this may be long long.
*/
typedef long lxref;
/**
read a raw isis record by xref.
the memory must be freed after usage.
*/
extern long *ldb_readRecAtOff ( int dbid, lxref off, long *nxtoff );
/*
** ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
** detailed search structures and functions
**
** ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
*/
enum {
LDB_MAX_KEYLEN = 31, /* actually, it's 30 for isis-1 */
LDB_TERMBUF = 128, /* longs */
LDB_INDEXES = 2
};
/**
the isis-1 posting is an 64bit number.
In the file it's in big endian order, to allow ordering using memcmp.
In the LdbP we have native order to use native long long.
Until we need more bits for postings, we may use the long long type
provided by gcc and some other compilers.
the 8 bytes are: highest 3 for the rowid, then 2 tag, then 3 pos.
the 3 pos bytes in turn are 1 byte for the occurence
and 2 bytes for the word count.
(e.g. value 2<<16 | 7 if term was found in the 7th word of
the 2nd occurence of field 'tag').
we usually don't care, since for a near condition we always
want a small distance w/in the same occurence which may be
conveniently checked by the difference of 3 byte pos values.
BUT NOTE: ... unfortunately we DO NEED one more bit for marking
postings during AND operations. we abuse the highest bit of the
word counter, since it's for sure useless for any reasonable
distance check.
*** BE CAREFULL WHEN COMPARING POSTINGS DURING "AND" OP ***
use the LDBP_xxM macros below.
*/
typedef union {
char bytes[8];
long long val;
} LdbP;
/* all macros operate on pointers */
/* access parts */
#define LDBP_ROW( p ) (0xffffffL & (long)((p)->val >> 40))
#define LDBP_TAG( p ) ( 0xffffL & (long)((p)->val >> 24))
#define LDBP_POS( p ) (0xff7fffL & (long)((p)->val))
#define LDBP_MARK( p ) ( 0x8000 & (long)((p)->val))
#define LDBP_SETMARK( p ) ((p)->val |= 0x8000LL)
#define LDBP_CLRMARK( p ) ((p)->val &= 0xffffffffffff7fffLL)
/* value w/o the mark bit */
#define LDBP_IGNMARK( p ) ((p)->val & 0xffffffffffff7fffLL)
/* compare according to rowid, tag, pos */
#define LDBP_EQ( a, b ) ((a)->val == (b)->val)
#define LDBP_GT( a, b ) ((a)->val > (b)->val)
/* compare IGNORING MARK according to rowid, tag, pos */
#define LDBP_EQM( a, b ) (LDBP_IGNMARK(a) == LDBP_IGNMARK(b))
#define LDBP_GTM( a, b ) (LDBP_IGNMARK(a) > LDBP_IGNMARK(b))
/* manipulation: set bottom / top for row of given posting */
#define LDBP_SETROWBOT( d, s ) ((d)->val = (s)->val & 0xffffff0000000000LL)
#define LDBP_SETROWTOP( d, s ) ((d)->val = (s)->val | 0xffffffffffLL)
/**
postings come in arrays with some header.
The standard structure gives a 8k buffer,
but member len may give other actual length.
*/
typedef struct {
short mode; /* in: merge flags */
short near; /* in: near distance; in OR mode: collect pos info */
long tag; /* in: tag, to which postings are restricted */
long skp; /* in: ignore mfns < skp */
long len; /* in: length (# of postings) of buffer (if 0 : default length) */
long fil; /* io: number of postings actually used */
long cut; /* io: min mfn ignored due to buffer length */
LdbP p[OPENISIS_SETLEN];
/**
8 bytes as in IFP file: mfn[3],tag[2],occ[1],cnt[2]
highest bit of cnt (1LL<<15) is used as mark
*/
} LdbPost;
enum {
LDB_OR,
LDB_AND,
LDB_NOT, /* like AND, but keep unmarked postings */
LDB_PFX = 4, /* prefix match */
LDB_KEEPMARKS = 8 /* do not compact after AND/NOT */
};
enum {
LDB_NEAR_F = 0x7fff, /* the (F): same occurence of field */
LDB_NEAR_G = -0x8000 /* the (G): same field */
};
/**
*/
extern int ldb_search ( int db, const char *key, LdbPost *post,
OpenIsisRec *rec );
extern long ldb_p2s ( OpenIsisSet *set, LdbPost *post );
enum {
LDB_ACTAB, /* alpha, != 0 for all "word" characters */
LDB_UCTAB, /* each characters uppercase code */
LDB__TABS
};
typedef unsigned char LdbCTab[256];
/* get a db's character tables */
extern LdbCTab *ldb_tabs( int db );
#define LDB_H
#endif /* LDB_H */
./lrec.c 0100644 0001130 0001130 00000035344 07550303061 010771 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
/*
$Id: lrec.c,v 1.22 2002/10/07 13:03:30 kripke Exp $
implementation of record cooking.
*/
#include
#include
#include /* vsnprintf */
#include /* free */
#include /* memset et al */
#ifdef WIN32
# define vsnprintf _vsnprintf
#endif /* WIN32 */
#include "ldb.h"
#include "lio.h"
/* ************************************************************
private types
*/
/* ************************************************************
private data
*/
/* ************************************************************
private functions
*/
#define FIXUP( r ) \
{ \
char * base = (char*)r; \
int i; \
for ( i=0; i < (r)->len; i++ ) \
(r)->field[i].val = base + (long)(r)->field[i].val; \
}
/* ************************************************************
package functions
*/
/* ************************************************************
public functions
*/
OpenIsisRec* openIsisClone ( OpenIsisRec *r, long room, int discard )
{
OpenIsisRec *nr = NULL;
long nfields, nbytes, hadfields, hadcontent;
if ( NULL == r ) {
nfields = 80; /* for a 1K base */
nbytes = 8*1024;
if ( nbytes < OPENISIS_BASESZ(80) + room*3/2 )
nbytes = OPENISIS_BASESZ(80) + room*3/2;
hadfields = 0;
hadcontent = 0;
} else {
hadfields = r->len;
hadcontent = r->used - r->base;
if ( 0 > room ) { /* shrink to fit */
nfields = r->len;
nbytes = OPENISIS_BASESZ(nfields) + hadcontent;
} else {
nfields = 54 > r->len ? 80 : (r->len * 3 / 2); /* add 50% */
if ( nfields < r->fields )
nfields = r->fields;
nbytes = 6*1024 > r->bytes ? 8*1024 : (r->bytes *3 / 2);
if ( nbytes < r->used + room )
nbytes = r->used + room*3/2;
nbytes += (nfields - r->fields)*sizeof(OpenIsisField);
}
}
LOG_DBG( LOG_DEBUG,
"extending rec size %ld (cont %ld) to %ld bytes, %ld -> %ld fields",
NULL==r ? -1 : r->bytes, hadcontent, nbytes, hadfields, nfields );
assert( nbytes >= OPENISIS_BASESZ( nfields ) + hadcontent + room );
if ( NULL == (nr = (OpenIsisRec *)lmalloc( nbytes )) )
return NULL;
memset( nr, 0, nbytes ); /* paranoia */
nr->bytes = nbytes;
nr->fields = nfields;
nr->base = OPENISIS_BASESZ( nfields );
nr->used = nr->base + hadcontent;
nr->len = hadfields;
if ( NULL != r ) {
char *obuf = ((char*)r) + r->base;
char *nbuf = ((char*)nr) + nr->base;
/* for NULL == r, the following are 0 by memset */
nr->dbid = r->dbid;
nr->rowid = r->rowid;
nr->state = r->state;
if ( hadcontent )
memcpy( nbuf, obuf, hadcontent );
if ( hadfields ) {
int i;
long displace = nbuf - obuf; /* ptrdiff_t */
OpenIsisField *f = nr->field;
memcpy( nr->field, r->field, hadfields*sizeof(OpenIsisField) );
for ( i=hadfields; i--; )
if ( NULL != f[i].val )
f[i].val += displace;
#ifndef NDEBUG
{
char *end = ((char*)nr) + nr->used;
for ( i=hadfields; i--; )
if ( NULL != f[i].val
&& (f[i].val < nbuf || f[i].val + f[i].len > end)
) {
int wasok = r->field[i].val >= obuf
&& r->field[i].val + r->field[i].len <= obuf + hadcontent;
log_msg( LOG_ERROR, "OOPS! nuked field %d which previously was %s",
i, wasok ? "ok" : "already broken" );
return NULL; /* no cleanup, we're nearly dead anyway */
}
}
#endif
}
if ( discard )
free( r );
}
assert( OPENISIS_RECOK(nr) );
return nr;
} /* openIsisClone */
OpenIsisRec* openIsisPrintf ( OpenIsisRec *r, int discard,
int tag, const char *fmt, ... )
{
char buf[1024];
int l;
va_list ap;
va_start( ap, fmt );
l = vsnprintf( buf, sizeof(buf), fmt, ap );
if ( 0 > l ) /* older versions return -1 */
l = sizeof(buf);
va_end( ap );
OPENISIS_ADD( r, tag, buf, l, discard );
return r;
} /* openIsisPrintf */
OpenIsisRec *openIsisReadRow ( int db, long rowid )
{
OpenIsisRec *r = (OpenIsisRec *) openIsisReadRaw( db, rowid );
if ( NULL == r )
return NULL;
FIXUP( r )
assert( OPENISIS_RECOK( r ) );
#if 1
return r;
#else
#if 1 /* clone test */
return openIsisClone( r, -1, !0 );
#else /* hurt me plenty test */
{
OpenIsisRec *nr = NULL;
int i;
OpenIsisField *f = r->field;
for ( i=0; ilen; i++ ) {
#if 0
OPENISIS_ADD( nr, f[i].tag, f[i].val, f[i].len, !0 );
#else /* even worse, but works: */
OPENISIS_ADD( nr, f[i].tag, NULL, 0, !0 );
OPENISIS_CAT( nr, f[i].val, f[i].len, !0 );
#endif
}
if ( NULL == nr )
OPENISIS_CHKFREE( nr, 0, 0 );
if ( NULL != nr ) {
nr->dbid = r->dbid;
nr->rowid = r->rowid;
nr->state = r->state;
}
return openIsisClone( nr, -1, !0 ); /* and re-shrink */
}
#endif
#endif
} /* openIsisReadRow */
OpenIsisRec *openIsisRead ( OpenIsisRec *buf,
const char *fmt, int db, long rowid )
{
OpenIsisRec *r = openIsisReadRow( db, rowid );
OpenIsisRec *q;
if ( NULL == r )
return NULL;
q = openIsisFmt( buf, fmt, r );
free( r );
return q;
} /* openIsisRead */
OpenIsisRec *openIsisScan ( int db, long rowid, long tag, const char *txt )
{
long max = openIsisMaxRowid( db );
int tlen = strlen( txt );
const char f = *txt;
for ( ; rowid <= max; rowid++ ) {
int i, found = 0;
OpenIsisRec *r = (OpenIsisRec *) openIsisReadRaw( db, rowid );
char *base = (char*)r;
if ( NULL == r )
continue;
for ( i=0; i < r->len; i++ ) {
const char *c, *e;
r->field[i].val = base + (long)r->field[i].val;
if ( found
|| (tag > 0 && tag != r->field[i].tag)
|| tlen > r->field[i].len
)
continue;
c = r->field[i].val;
e = c + r->field[i].len - tlen;
for ( ; c<=e; c++ )
if ( f == *c && !memcmp( c, txt, tlen ) ) {
found = !0;
break;
}
}
if ( found )
return r;
}
return NULL;
} /* openIsisScan */
OpenIsisRec *openIsisReadField ( OpenIsisRec *r, const OpenIsisField* field )
{
long i;
long nfields = 0; /* number of subfields */
long size; /* byte length of field list */
const char *p, *e;
if ( NULL == field || 2 > field->len || '^' != field->val[0] )
return NULL;
/* go counting hats ... */
for ( e = (p=field->val) + field->len - 1; plen, field->val );
if ( NULL != r ) {
if ( 1 > r->len )
return NULL;
size = sizeof(OpenIsisRec) + (r->len - 1) * sizeof(OpenIsisField);
if ( nfields > r->len )
nfields = r->len;
} else {
/* first field already counted in sizeof(OpenIsisRec) */
size = sizeof(OpenIsisRec) + (nfields - 1) * sizeof(OpenIsisField);
r = (OpenIsisRec *)lmalloc( size );
if ( NULL == r )
return NULL;
}
memset( r, 0, size );
r->rowid = 0;
r->len = nfields;
e = (p = field->val) + field->len;
p++;
for ( i=0; i= e ) {
log_msg( LOG_ERROR, "confused cats hound counting hats at %d in '%.*s'",
i, field->len, field->val );
r->len = i;
break;
}
/* p is after a hat */
r->field[ i ].tag = (unsigned char)*p++;
r->field[ i ].val = p;
/* advance in buf */
while ( pfield[ i ].len = p - r->field[ i ].val;
}
return r;
} /* openIsisReadField */
enum { /* stream reading state */
RS_REC, /* not in any record */
RS_TAG, /* in tag: done is at beginning of line */
RS_SEP, /* in sep: done is at beginning of sep */
RS_VAL, /* done is somewhere in the value ... */
RS_EOL /* not really a state, but a flag */
};
enum { /* character type */
CT_PR, /* ordinary printable */
CT_HT, /* intra-field separator ("horizontal tab") */
CT_LF, /* field separator ("line feed") */
CT_FF /* record separator ("form feed") */
};
static unsigned char ctype[256] = {
/* NUL STX SOT ETX EOT ENQ ACK BEL */
CT_LF,CT_HT,CT_HT,CT_HT,CT_HT,CT_HT,CT_HT,CT_HT,
/* BS HT LF VT FF CR SO SI */
CT_HT,CT_HT,CT_LF,CT_HT,CT_FF,CT_LF,CT_HT,CT_HT,
/* DLE DC1 DC2 DC3 DC4 NAK SYN ETB */
CT_HT,CT_HT,CT_HT,CT_HT,CT_HT,CT_HT,CT_HT,CT_HT,
/* CAN EM SUB ESC FS GS RS US */
CT_HT,CT_HT,CT_HT,CT_HT,CT_HT,CT_FF,CT_LF,CT_HT,
/*blank ! " # $ % & ' */
CT_HT,CT_HT,CT_HT, 0 , 0 ,CT_HT,CT_HT,CT_HT,
/* ( ) * + , - . / */
CT_HT,CT_HT,CT_HT,CT_HT,CT_HT, 0 , 0 ,CT_HT,
/* 0 1 2 3 4 5 6 7 */
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
/* 8 9 : ; < = > ? */
0 , 0 ,CT_HT, 0 ,CT_HT,CT_HT,CT_HT,CT_HT,
/* @ A B C D E F G */
CT_HT, 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , /* H-O */
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , /* P-W */
/* X Y Z [ \ ] ^ _ */
0 , 0 , 0 , 0 ,CT_HT, 0 ,CT_HT, 0 ,
/* ` a b c d e f g */
CT_HT, 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
/* x y z { | } ~ DEL */
0 , 0 , 0 ,CT_HT,CT_HT,CT_HT,CT_HT,CT_HT
/* upper half assumed printable */
};
/**
read a record from a stream.
This may either
read a record and return 1
not get enough bytes on a non-blocking stream and return 0
have some error and return negative
*/
int openIsisReadStream ( OpenIsisRecStream *s )
{
LioStream *lio = s->in;
int state = 3 & s->flg;
int emptyline = 0;
int ret = 0;
if ( NULL == s->rec
&& NULL == (s->rec = s->buf) /* have buffer ? */
&& NULL == (s->rec = openIsisClone( NULL, 0, 0 ))
/* allocate and prepare an 8k standard record */
) {
ret = log_msg( LOG_ERROR, "could not get record" );
goto fatal;
}
if ( RS_REC == state ) { /* initialize */
s->rec->rowid = 0;
s->rec->len = 0; /* nuke all fields */
s->rec->used = s->rec->base; /* nuke value buffer */
/* let base/fields untouched, it hopefully has a reasonable value */
state = RS_TAG;
}
if ( ! s->rec->len )
state = RS_TAG;
for (;;) {
unsigned char *b, *p, *e; /* begin, pointer, end */
if ( lio->done >= lio->fill ) {
reload:
if ( ! lio_sread( lio ) )
goto done; /* EAGAIN -- see ya later */
/* so we have bytes or EOF */
if ( lio->done >= lio->fill ) {
if ( !(LIO_EOF & lio->flg) ) {
log_msg( LOG_ERROR, "confused in instream: no bytes and no EOF" );
lio_close( lio->fd );
lio->fd = -1;
lio->flg |= LIO_EOF;
}
break;
}
}
/* so we have bytes */
p = b = lio->b + lio->done;
e = lio->b + lio->fill;
/* try to eat one line */
switch ( state ) {
case RS_TAG: {
int tag = -1;
int recognized = 0;
if ( 2 & ctype[*p] ) { /* line starts with line/record sep */
if ( '\n' == *p && (RS_EOL & s->flg) ) { /* TODO */
s->flg &= ~RS_EOL;
continue;
}
emptyline = !0;
break;
}
for (;;p++) {
if ( p < e ) {
if ( ! ctype[*p] )
continue; /* tight loop */
break;
}
if ( 4096 > p - b )
goto reload;
log_msg( -LERR_TRASH, "tag too long" );
goto fatal;
}
/* identify tag between s and p */
if ( p == b )
recognized = !0;/* empty tag */
else if ( '0' <= *b && *b <= '9' ) {
unsigned char *d = b; /* digit ? */
tag = 0;
while ( d < p && '0' <= *d && *d <= '9' )
tag = 10*tag + *d++ - '0';
if ( !(recognized = d == p) ) /* all digits */
tag = -1;
}
if ( ! recognized && NULL != s->dict ) { /* dict lookup */
int i=s->dict->len, l = p-b;
OpenIsisField *f = s->dict->field;
for ( ; i--; f++ )
if ( l == f->len && memcmp( b, f->val, l ) ) {
tag = f->tag;
break;
}
}
/* create new field unless it's a continuation line */
if ( p == b && s->rec->len )
;
else if ( recognized )
OPENISIS_ADD( s->rec, tag, NULL, 0, s->rec != s->buf );
else
OPENISIS_ADD( s->rec, tag, b, p-b, s->rec != s->buf );
if ( NULL == s->rec )
goto outamem;
/* did that */
lio->done = (b = p) - lio->b;
if ( 2 & ctype[*p] )
break;
state = RS_SEP;
} case RS_SEP:
/* p = b is at beginning of separator */
while ( p < e && ' ' == *p ) /* skip leading blanks */
p++;
if ( p < e && CT_HT == ctype[*p] && '\t' != *p++ )
while ( p < e && ' ' == *p ) /* skip trailing blanks */
p++;
if ( e == p ) {
if ( 4096 > p - b )
goto reload;
log_msg( -LERR_TRASH, "sep too long" );
goto fatal;
}
assert( s->rec->len );
if ( s->rec->field[s->rec->len - 1].len )
OPENISIS_CAT( s->rec, b, p-b, s->rec != s->buf );
if ( NULL == s->rec )
goto outamem;
lio->done = (b = p) - lio->b;
if ( 2 & ctype[*p] )
break;
state = RS_VAL;
case RS_VAL: default: /* ??? */
while ( p < e && !(2 & ctype[*p]) )
p++;
assert( s->rec->len );
if ( p > b )
OPENISIS_CAT( s->rec, b, p-b, s->rec != s->buf );
if ( NULL == s->rec )
goto outamem;
lio->done = (b = p) - lio->b;
if ( p == e )
goto reload;
/* else we found line or record separator */
} /* switch state */
if ( '\r' == *p ) { /* do that ole' ugly CR/NL quatsch */
if ( e-p < 2 && 0 < lio_sread(lio) ) /* extend end */
e = lio->b + lio->fill;
if ( e-p < 2 )
s->flg |= RS_EOL; /* possibly check later */
else if ( '\n' == p[1] )
p++;
}
lio->done = 1 + p - lio->b; /* consider p eaten */
if ( CT_FF == ctype[*p]
|| (emptyline && (OPENISIS_STOPONEMPTY & s->flg))
)
break;
state = RS_TAG;
}
/* got record */
state = RS_REC;
if ( NULL != s->rec && s->rec->len ) {
if ( (OPENISIS_AUTOCLONE & s->flg) && s->rec == s->buf )
s->rec = openIsisClone( s->buf, 0, 0 );
ret = 1;
}
goto done;
outamem:
fatal:
if ( LIO_SISOPEN( s->in ) )
lio_sclose( s->in );
done:
s->flg = state | (~3 & s->flg); /* save state */
return ret;
} /* openIsisReadStream */
./lstr.c 0100644 0001130 0001130 00000023131 07447676140 011037 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
/*
$Id: lstr.c,v 1.11 2002/03/25 19:36:44 kripke Exp $
implementation of record cooking.
*/
#include
#include "lstr.h"
#include "luti.h"
/* ************************************************************
private types
*/
/* ************************************************************
private data
*/
static const char * nameMfc[] = {
"Mfc", "CTLM", "NMFN", "NMFB", "NMFP", "TYPE", "RCNT", "MFX1", "MFX2", "MFX3"
};
static long descMfc[] = {
LSTRSIZE( 9, 0, 0 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT,
LMBRINT,
LMBRINT,
LMBRSHORT,
LMBRSHORT,
LMBRINT,
LMBRINT,
LMBRINT,
LMBRINT
}; /* descMfc */
static const char * nameMfr[] = {
"Mfr", "MFN", "RECL", "BWB", "BWP", "BASE", "NVF", "STAT",
"TAG", "'POS", "LEN"
};
static long descMfr[] = {
LSTRSIZE( 7, 3, 0 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT,
LMBRSHORT,
LMBRINT,
LMBRSHORT,
LMBRSHORT,
LMBRSHORT,
LMBRSHORT,
LMBRSHORT,
LMBRSHORT,
LMBRSHORT
}; /* descMfr */
/* aligned version */
static long descMfrA[] = {
LSTRSIZE( 7, 3, 0 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT,
LMBRSHORT,
LSTRLOFF( LMBRINT, 8 ),
LMBRSHORT,
LMBRSHORT,
LMBRSHORT,
LMBRSHORT,
LMBRSHORT,
LMBRSHORT,
LMBRSHORT
}; /* descMfr */
static const char * nameXrf[] = {
"Xrf", "XPOS", "XREC"
};
static long descXrf[] = {
LSTRSIZE( 1, 1, 127 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT,
LMBRINT
}; /* descXrf */
static const char ** nameMst[LSTR_MST] = {
nameMfc, nameMfr, nameXrf
};
static long* descMst[LSTR_MST] = {
descMfc, /* MST head */
descMfr, /* MST record */
descXrf /* XRF record */
};
static long* descMstA[LSTR_MST] = {
descMfc, /* MST head */
descMfrA, /* MST record */
descXrf /* XRF record */
};
static const char * nameCnt[] = {
"Cnt", "TYPE", "ORDN", "ORDF", "N", "K", "LEV", "POSR", "NMAX", "FMAX", "ABNO"
};
static long descCnt[] = {
LSTRSIZE( 10, 0, 0 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRSHORT, /* TYPE */
LMBRSHORT, /* ORDN */
LMBRSHORT, /* ORDF */
LMBRSHORT, /* N */
LMBRSHORT, /* K */
LMBRSHORT, /* LEV */
LMBRINT, /* POSR */
LMBRINT, /* NMAX */
LMBRINT, /* FMAX */
LMBRSHORT /* ABNO */
}; /* descCnt */
static long descCntA[] = {
LSTRSIZE( 10, 0, 0 ), /* sizes (fix, rep, occ) */
0,28,44,
LMBRSHORT, /* TYPE */
LMBRSHORT, /* ORDN */
LMBRSHORT, /* ORDF */
LMBRSHORT, /* N */
LMBRSHORT, /* K */
LMBRSHORT, /* LEV */
LMBRINT, /* POSR */
LMBRINT, /* NMAX */
LMBRINT, /* FMAX */
LMBRSHORT /* ABNO */
}; /* descCntA */
static const char * nameN0[] = {
"N0", "POS", "OCK", "TYPE", "'KEY", "REF"
};
static long descN01[] = {
LSTRSIZE( 3, 2, 10 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT, /* POS */
LMBRSHORT, /* OCK */
LMBRSHORT, /* TYPE */
10, /* KEY */
LMBRINT /* REF */
};
static long descN01A[] = {
LSTRSIZE( 3, 2, 10 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT, /* POS */
LMBRSHORT, /* OCK */
LMBRSHORT, /* TYPE */
10, /* KEY */
LSTRLOFF( LMBRINT, 12 ) /* REF */
};
static long descN02[] = {
LSTRSIZE( 3, 2, 10 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT, /* POS */
LMBRSHORT, /* OCK */
LMBRSHORT, /* TYPE */
30, /* KEY */
LMBRINT /* REF */
};
static long descN02A[] = {
LSTRSIZE( 3, 2, 10 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT, /* POS */
LMBRSHORT, /* OCK */
LMBRSHORT, /* TYPE */
30, /* KEY */
LSTRLOFF( LMBRINT, 32 ) /* REF */
};
static const char * nameL0[] = {
"L0", "POS", "OCK", "TYPE", "PS", "'KEY", "INFB", "INFP"
};
static long descL01[] = {
LSTRSIZE( 4, 3, 10 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT, /* POS */
LMBRSHORT, /* OCK */
LMBRSHORT, /* TYPE */
LMBRINT, /* PS */
10, /* KEY */
LMBRINT, /* INFB */
LMBRINT /* INFP */
};
static long descL01A[] = {
LSTRSIZE( 4, 3, 10 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT, /* POS */
LMBRSHORT, /* OCK */
LMBRSHORT, /* TYPE */
LMBRINT, /* PS */
10, /* KEY */
LSTRLOFF( LMBRINT, 12 ), /* INFB */
LMBRINT /* INFP */
};
static long descL02[] = {
LSTRSIZE( 4, 3, 10 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT, /* POS */
LMBRSHORT, /* OCK */
LMBRSHORT, /* TYPE */
LMBRINT, /* PS */
30, /* KEY */
LMBRINT, /* INFB */
LMBRINT /* INFP */
};
static long descL02A[] = {
LSTRSIZE( 4, 3, 10 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT, /* POS */
LMBRSHORT, /* OCK */
LMBRSHORT, /* TYPE */
LMBRINT, /* PS */
30, /* KEY */
LSTRLOFF( LMBRINT, 32 ), /* INFB */
LMBRINT /* INFP */
};
static const char * nameIfp[] = {
"IfH", "NXTB", "NXTP", "TOTP", "SEGP", "SEGC", "MFN", "INFO"
};
static long descIfp[] = {
#ifdef CONVERT_POSTINGS
/*
to properly convert postings,
the repeated part needs to be a big-endian 64bit (long long) int
however, converting can't be done in a straightforward fashion anyway,
since posting segments have holes of one or two longs at block boundaries
*/
LSTRSIZE( 5, 2, 1 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
LMBRINT, /* */
LMBRINT, /* */
#else
LSTRSIZE( 5, 0, 0 ), /* sizes (fix, rep, occ) */
LSTR_AUTOLENGTHS,
#endif
LMBRINT, /* */
LMBRINT, /* */
LMBRINT, /* */
LMBRINT, /* */
LMBRINT /* */
};
static const char ** nameInv[LSTR_INV] = {
nameCnt,
nameN0,
nameL0,
nameN0,
nameL0,
nameIfp
};
static long* descInv[LSTR_INV] = {
descCnt, /* CNT record */
descN01, /* N01 record */
descL01, /* L01 record */
descN02, /* N02 record */
descL02, /* L02 record */
descIfp /* Ifp record */
};
static long* descInvA[LSTR_INV] = {
descCntA, /* CNT record */
descN01A, /* N01 record */
descL01A, /* L01 record */
descN02A, /* N02 record */
descL02A, /* L02 record */
descIfp /* Ifp record */
};
/* ************************************************************
private functions
*/
/* ************************************************************
package data
*/
const LstrSet lstrlib[LSET_SETS] = {
{ /* Isis 1 MST */
nameMst, { descMst, descMstA }
},
{ /* Isis 1 INV */
nameInv, { descInv, descInvA }
},
}; /* lstrlib */
/* ************************************************************
package functions
*/
/* ************************************************************
public functions
*/
int lstr_auto ( long *str ) {
static int pow2[] = { 1, 2, 4, 8 };
int ret = 0;
int fix,rep,occ,i;
long *xmbrs, txlen, tilen;
int nmbrs, off = 0, xlen[2] = {0,0}, blen[2] = {0,0}, x=0;
if ( NULL == str ) {
#define DOALL( x ) \
for ( i=0; i<(int)(sizeof(x)/sizeof(x[0])); i++ ) \
if ( NULL != x[i] ) ret |= lstr_auto( x[i] )
DOALL( descMst );
DOALL( descInv );
DOALL( descMstA );
DOALL( descInvA );
return ret;
}
fix = LSTRFIX(*str);
rep = LSTRREP(*str);
occ = LSTROCC(*str);
xmbrs = str + LSTR_XMBR;
nmbrs = fix;
for ( x=0; ; x++ ) { /* fix and rep */
for ( i=0; i> 16) )
str[LSTR_XRLO] |= xlen[0] << 16;
else if ( xlen[0] > (short)(str[LSTR_XRLO] >> 16) )
log_msg( LOG_WARN, "short xlen[0] %hd < %d",
(short)(str[LSTR_XRLO] >> 16), xlen[0] );
if ( !(short)str[LSTR_XRLO] )
str[LSTR_XRLO] |= 0xffff & xlen[1];
else if ( xlen[1] > (short)str[LSTR_XRLO] )
log_msg( LOG_WARN, "short xlen[1] %hd < %d",
(short)str[LSTR_XRLO], xlen[1] );
/* build totals applying occ */
txlen = (short)(str[LSTR_XRLO] >> 16)
+ (long)occ*(short)str[LSTR_XRLO];
if ( !str[LSTR_XLEN] )
str[LSTR_XLEN] = txlen;
else if ( txlen > str[LSTR_XLEN] )
log_msg( LOG_WARN, "short xlen %ld < %ld",
str[LSTR_XLEN], txlen );
tilen = LSTRLEN(*str) + blen[0] + (long)occ*blen[1];
if ( !str[LSTR_ILEN] )
str[LSTR_ILEN] = tilen;
else if ( tilen > str[LSTR_ILEN] )
log_msg( LOG_WARN, "short ilen %ld < %ld",
str[LSTR_ILEN], tilen );
LOG_DBG( LOG_ALL, "%08lx xrlo %08lx xl %ld il %ld",
str[0], str[1], str[2], str[3] );
return ret;
} /* lstr_auto */
./lstr.h 0100644 0001130 0001130 00000022645 07546374256 011060 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
#ifndef LSTR_H
/*
$Id: lstr.h,v 1.13 2002/09/30 16:21:01 kripke Exp $
definition of ISIS internal structures for openisis lib implementation.
This header defines the common internal data structures and their members
and external structure and member definition types.
A internal data structure is the common in-memory representation of
ISIS data; it may be backed by different external (in-file) structures.
An in-memory data structure
is a contigous long* chunk of memory, consisting of the sizes (one long),
an array of members (long) and some buffer (array of char).
A data structures sizes consists of the lengths
fix and rep of the fixed and repeated part, resp.,
and the number of occurences of the repeated part.
It is used in both an abstract description of a data type
and each instance of a data structure.
If the number of occurences of the repeated part is not fixed,
it is 0 in the abstract case.
The lengths are suitable as offsets into the lmbr array.
This structure is designed to fit within 32 bits.
The sizing is the only common meta-data description of a logical
data type. For readability of code, however, the member indexes
are enum-erated. When actually used, the type has to match the code's
assumptions anyway.
The numbers fix and rep are fixed for a given data structure.
Although the logical data type has more properties,
especially the type of each member (numeric, C-string or raw mem),
a representation of this data is needed only when converting
external data and thus is stored (somewhat redundantly)
with the external properties.
There are only two types of members:
numeric and memory (mostly some character data).
both are represented by a long, which, for memory,
denotes an offset (counted in BYTES, not longs) from the records start
(NOT offset into the data buffer, as with the isis mst record).
The member array consists of a fixed number fix of initial
members and a number occ of occurences
of member arrays of fixed length rep.
While occ is usually fixed,
it varies for masterfile records.
Note that this layout somewhat resembles the isis record itself,
which supports a single level of nesting and repetition.
*/
/* id of a structure */
/* construction
*/
#define LSTRID( set, no, vari ) \
(((long)(vari))<<16 | (set)<<8 | (no) )
/* access
*/
#define LSTRNO( l ) (char)(l) /* no of record type in set */
#define LSTRSET( l ) (char)((l)>>8) /* record set number */
#define LSTRVAR( l ) (short)((l)>>16) /* variant */
/* size of a structure */
/* construction
*/
#define LSTRSIZE( fix, rep, occ ) \
(((long)(occ))<<16 | (rep)<<8 | (fix))
/* access
*/
#define LSTRFIX( l ) (char)(l) /* length of fixed part */
#define LSTRREP( l ) (char)((l)>>8) /* length of repeated part */
#define LSTROCC( l ) (short)((l)>>16) /* the members */
/* actual number of members */
#define LSTRMEMS( l ) (LSTRFIX(l) + LSTRREP(l)*LSTROCC(l))
/* in-memory size of members (w/o buffer). */
#define LSTRLEN( t ) ((long)sizeof(long)*(1 + LSTRMEMS(t)))
/**
Description of a external members layout is encoded as a short "len"
and a short offset "off" into the data or repeated part, resp.
offset and len are combined into one long.
If offsets are 0, they are computed based on the assumption,
that members are layouted packed in the declared ordering.
For character data, len just gives the length of the member
in bytes (up to 2**15), with the highest bit cleared.
A length of 0 denotes character data of variable size.
Numerical data is marked with the highest bit set.
For numbers, the lowest two bits give the ld of the length in bytes
(i.e. 0=1,1=2,2=4,3=8 bytes).
The offset and length of the actually used bits within these bytes
are each encoded in six bits, which is enough for 0..63.
A length of 0 (as resulting from 6-bit overflow on '64') means no masking.
The 15th bit, if set, denotes high endian data.
Note that the only reason for stuffing all that into a short is,
that it is sufficient. The amount of space saved is minimal and
depending on the actual processor there may be a slight performance
penalty for the bit operations.
*/
#define LONG2LEN( l ) ((short)(l))
#define LONG2OFF( l ) ((short)((l) >> 16))
/* constants
*/
#define LMBR_FNUM 0x8000 /* flag numeric data */
#define LMBR_FHIE 0x4000 /* flag high endian data */
/* construction
*/
#define LMBRNUM( ld ) (LMBR_FNUM | ld)
/* standard nueric types: the world according to Java :) */
#define LMBRBYTE LMBRNUM(0)
#define LMBRSHORT LMBRNUM(1)
#define LMBRINT LMBRNUM(2)
#define LMBRLONG LMBRNUM(3)
/* bit mask construction for numerical data */
#define LMBRBITS( off, len, ld ) (LMBR_FNUM | (off)<<8 | (len)<<2 | (ld))
#define LMBRBITS4( off, len ) (LMBR_FNUM | (off)<<8 | (len)<<2 | 2)
/* if you have to specify the offset explicitly */
#define LSTRLOFF( len, off ) ((len) | (off)<<16)
/* access
*/
#define LMBRISNUM( f ) (LMBR_FNUM & (f))
#define LMBRLD( f ) (0x3 & (f))
/* bit access for numerical data */
#define LMBRISBITS( f ) (0x3ffc & (f))
#define LMBRBITLEN( f ) (0x003f & ((f) >> 2))
#define LMBRBITOFF( f ) (0x003f & ((f) >> 8))
/**
Description of an external structure layout.
This is usually fixed for a given (implementation of a) type;
for the isis mst record, however, the occurences of the repeated
part and the total external and internal lengths vary for each record.
*/
enum {
LSTR_SIZE, /* sizes */
LSTR_XRLO, /* repeated part byte length and offset <<16; auto */
LSTR_XLEN, /* ext. total length of data; auto */
LSTR_ILEN, /* int. total length of data; auto */
LSTR_XMBR /* members */
};
#define LSTR_AUTOLENGTHS 0,0,0
#define LSTR_LONGS( totmbrs ) (4+(totmbrs))
/* technical variants of a record */
typedef enum {
LVAR_PAC, /* packed little endian base variant */
LVAR_ALI, /* aligned variant */
LVAR_BIG, /* big endian aligned */
LVAR_VARI
} lstr_variant;
/* known record sets */
enum {
LSET_MST, /* isis 1 master file */
LSET_INV, /* isis 1 inverted file */
LSET_SETS
};
typedef struct {
const char ***name; /* an array of names per record type */
long **desc[LVAR_VARI]; /* an array of descs per variant */
} LstrSet;
/* data types of ISIS 1 master file */
typedef enum {
LSTR_MFC, /* MST head */
LSTR_MFR, /* MST record */
LSTR_XRF, /* XRF record */
LSTR_MST /* count */
} lstr_mst;
enum { /* MFC members */
LMFC_CTLM = 1,
LMFC_NMFN, /* next available MFN */
LMFC_NMFB, /* next free block in MF counting from 1 */
LMFC_NMFP, /* next free pos in MF */
LMFC_TYPE, /* "0 for user's bases; 1 for messages" */
LMFC_RCNT, /* record count ? not used by CISIS */
LMFC_MFX1,
LMFC_MFX2, /* LAN lock */
LMFC_MFX3, /* LAN lock */
LMFC__FL
};
enum { /* XRF members */
LXRF_XPOS = 1,
LXRF_XREC
};
/* data types of ISIS 1 inverted file */
typedef enum {
LSTR_CNT, /* CNT record */
LSTR_N01, /* N01 record */
LSTR_L01, /* L01 record */
LSTR_N02, /* N02 record */
LSTR_L02, /* L02 record */
LSTR_IFP, /* IFP record */
LSTR_INV /* count */
} lstr_inv;
enum { /* CNT members */
LCNT_TYPE = 1, /* 1 or 2: N0x */
LCNT_ORDN, /* fix 5 */
LCNT_ORDF, /* fix 5 */
LCNT_N, /* fix 15 */
LCNT_K, /* fix 5 */
LCNT_LEV, /* depth of index */
LCNT_POSR, /* position of root */
LCNT_NMAX,
LCNT_FMAX,
LCNT_ABNO,
LCNT__FL
};
enum { /* N0x members */
LN0X_POS = 1, /* record number */
LN0X_OCK, /* number of keys */
LN0X_TYPE, /* type 1 or 2 */
LN0X__FL, /* offset of repeated part */
LN0X_KEY = 0, /* key chars */
LN0X_REF, /* ref to next node (>0) or leaf (<0) */
LN0X__RL /* length of repeated part */
};
enum { /* L0x members */
LL0X_POS = 1, /* record number */
LL0X_OCK, /* number of keys */
LL0X_TYPE, /* type 1 or 2 */
LL0X_PS, /* type 1 or 2 */
LL0X__FL, /* offset of repeated part */
LL0X_KEY = 0, /* key chars */
LL0X_INFB, /* ifp block */
LL0X_INFP, /* ifp pos */
LL0X__RL /* length of repeated part */
};
enum { /* IFP members */
LIFP_NXTB = 1, /* block of next segment */
LIFP_NXTP, /* pos of next segment */
LIFP_TOTP, /* total postings */
LIFP_SEGP, /* postings in this segment */
LIFP_SEGC, /* postings in this segment */
LIFP__FL, /* offset of repeated part */
LIFP_MFN = 0, /* mfn */
LIFP_ADD, /* additional qualifyers */
LIFP__RL /* length of repeated part */
};
extern const LstrSet lstrlib[LSET_SETS];
/** auto-initialise the given structure or all builtins, iff NULL.
*/
extern int lstr_auto ( long *str );
#define LSTR_H
#endif /* LSTR_H */
./luti.c 0100644 0001130 0001130 00000016720 07546641101 011023 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
/*
$Id: luti.c,v 1.13 2002/10/02 19:00:17 kripke Exp $
implementation of record cooking.
*/
#include
#include
#include /* malloc */
#include /* errno */
#include /* strerror */
#include /* isprint */
#include "lstr.h"
#include "luti.h"
/* ************************************************************
private types
*/
/* ************************************************************
private data
*/
log_level log_lev = LOG_ERROR;
static FILE *logout = NULL;
/* ************************************************************
private functions
*/
/* ************************************************************
package functions
*/
int lerrno ( int errno_ )
{
switch ( errno_ ) {
case EINTR:
case EAGAIN:
return -LERR_OK;
case EFAULT: /* structurally bad address */
case ENAMETOOLONG:
case ELOOP:
return -LERR_FAULT;
case EBADF: /* file "does not exist" */
case ENOENT:
case ENOTDIR:
return -LERR_BADF;
case EIO: /* file "is not accessible" */
case EEXIST:
case EISDIR:
case EACCES:
case ENXIO:
case ENODEV:
case EROFS:
case ETXTBSY:
case ENOSPC:
case EPIPE:
case ESPIPE:
return -LERR_IO;
case ENOMEM:
case EMFILE:
case ENFILE:
return -LERR_NOMEM;
case EBUSY:
return -LERR_BUSY;
}
return -LERR_INVAL;
} /* lerrno */
int log_msg ( int code, const char *fmt, ... )
{
va_list ap;
int level;
if ( 0 < code )
switch ( level = code ) {
case LOG_FATAL:
case LOG_ERROR:
code = -1;
break;
case LOG_IOERR:
if ( EAGAIN == errno || EINTR == errno ) /* no error */
return 0;
case LOG_SYSERR:
code = lerrno( errno );
break;
default:
code = 0;
}
else if ( -LERR_TRASH >= code )
level = LOG_FATAL;
else if ( -LERR_IO >= code )
level = LOG_SYSERR;
else
level = LOG_ERROR;
if ( level > log_lev )
return code;
if ( NULL == logout )
logout = stderr;
va_start( ap, fmt );
vfprintf( logout, fmt, ap );
va_end( ap );
if ( LOG_SYSERR == level || LOG_IOERR == level ) {
fputs( ": ", logout );
fputs( strerror(errno), logout );
}
fputc( '\n', logout );
return code;
} /* log_msg */
void log_rec ( log_level level, long *rec, const char **desc )
{
int occ = -1;
int nmbrs = LSTRFIX(*rec);
long *mbr = rec+1;
char *base = (char*)rec;
if ( level > log_lev || NULL == desc )
return;
if ( NULL == logout )
logout = stderr;
fprintf( logout, "record %.20s\n", *desc++ );
/* dump the fixed part (occ==-1) and each occurrence of repeated part. */
for ( ;/* occ < LSTROCC(*dst) */; ) { /* dump one part */
int i;
for ( i=0; i= LSTROCC(*rec) )
break;
if ( ! occ ) { /* was the fixed part, setup for repeated */
nmbrs = LSTRREP(*rec);
desc += i;
}
}
} /* log_rec */
void log_hex ( log_level level, const void *mem, int len )
{
const char *p = (const char *)mem;
int i = 0;
if ( level > log_lev )
return;
if ( NULL == logout )
logout = stderr;
for ( ; i 16 )
left = 16;
for ( ; j < left; j++ ) {
fprintf( logout, (3 == j%4) ? "%02x " : "%02x", p[j] );
}
for ( ; j < 16; j++ ) {
fputs( (3 == j%4) ? " " : " ", logout );
}
for ( j=0; j < left; j++ ) {
fputc( isprint(p[j]) ? p[j] : '.', logout );
}
fputc( '\n', logout );
}
} /* log_hex */
void * lmalloc ( long siz )
{
void *mem = malloc( siz );
if ( NULL == mem ) {
fputs( "out of memory\n", stderr );
if ( stderr != logout )
fputs( "out of memory\n", logout );
fprintf( logout, "when allocating %ld bytes: %s\n",
siz, strerror(errno) );
}
return mem;
} /* lmalloc */
/* ************************************************************
public functions
*/
void openIsisLog ( int level, const char *filename )
{
if ( ' ' <= level ) { /* by ascii value */
switch ( level ) {
case '-': level = LOG_NOCHANGE; break;
case 'o': level = LOG_OFF; break;
case 'f': level = LOG_FATAL; break;
case 's': level = LOG_IOERR /*LOG_SYSERR*/; break;
default:
if ( '0' <= level && level <= '9' ) {
level -= '0';
break;
}
case 'e': level = LOG_ERROR; break;
case 'w': level = LOG_WARN; break;
case 'i': level = LOG_INFO; break;
case 'v': level = LOG_VERBOSE; break;
case 'd': level = LOG_DEBUG; break;
case 't': level = LOG_TRACE; break;
case 'a': level = LOG_ALL; break;
}
}
if ( level != LOG_NOCHANGE )
log_lev = (log_level)level;
if ( NULL != filename )
if ( NULL == (logout = fopen( filename, "a" )) ) {
logout = stderr;
log_msg( 0, "could not open logfile '%s'\n", filename );
}
} /* openIsisLog */
char *openIsis2Html ( const char *str, long len )
{
long l;
char *p, *e, *ret;
if ( NULL == str )
return NULL;
if ( 0 > len )
len = strlen(str);
l = len + 1;
for ( e = (p=(char*)str) + len; p': l += 3; break;
case '&': l += 4; break;
case '"': l += 5; break;
/* do not replace ' w/ ' -- many clients don't grok it */
}
ret = lmalloc( l );
if ( NULL == ret )
return NULL;
for ( p=ret; str': *p='&'; p[1]='g'; p[2]='t'; p[3]=';'; p+=4; break;
case '&': *p='&'; p[1]='a'; p[2]='m'; p[3]='p'; p[4]=';'; p+=5; break;
case '"': *p='&';p[1]='q';p[2]='u';p[3]='o';p[4]='t';p[5]=';';p+=6;break;
default: *p++ = *str;
}
*p = 0;
return ret;
} /* openIsis2Html */
int openIsisValidUTF8 ( void *mem, int len, int *tof )
{
unsigned char *c = (unsigned char *)mem;
int f = 0; /* expected followers */
int l = len;
if ( NULL != tof )
f = *tof;
for ( ; l--; c++ ) {
unsigned char u = *c;
int tofollow = u < 128 ? 0 : u < 192 ? -1 : u < 224 ? 1 : 2;
if ( f ) {
if ( 0 > tofollow ) { f--; continue; }
break;
}
if ( 0 <= tofollow ) { f = tofollow; continue; }
goto croak;
}
if ( NULL != tof )
*tof = f;
if ( ! f )
return 0;
if ( 0 > l && NULL != tof ) /* exhausted buf, want follower */
return 0;
log_msg( LOG_ERROR, "expected follower at %d got %x", len-l-1, (int)*c );
return len-l;
croak:
log_msg( LOG_ERROR, "expected no follower at %d got %x", len-l-1, (int)*c );
return len-l;
}
./luti.h 0100644 0001130 0001130 00000007671 07546641101 011035 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
#ifndef LUTI_H
/*
$Id: luti.h,v 1.10 2002/10/02 19:00:17 kripke Exp $
common utilities for openisis lib implementation.
*/
#include
#include "openisis.h"
typedef enum { /* our very own errno */
LERR_OK, /* 0 is no error, also read(2)'s EINTR, EAGAIN */
LERR_EOF, /* end of file */
LERR_FAULT, /* NULL pointer or bad sized buffer given */
LERR_INVAL, /* general invalid parameters, any EINVAL errno */
LERR_BADF, /* bad file, also read(2)'s EINVAL, some of open(2) */
LERR_IO, /* real IO error, also write(2)'s ENOSPC, EPIPE */
LERR_NOMEM, /* out of memory, also open(2)'s EMFILE, ENFILE */
LERR_BUSY, /* object is busy */
LERR_TRASH, /* database internal consistency */
LERR_IDIOT, /* caught programming error */
LERR_ERRORS
} lerr;
typedef enum {
LOG_NOCHANGE = -1, /* leave as is when initialising log */
LOG_OFF = 0, /* don't log anything */
LOG_FATAL, /* fatal internal errors: we can't go on */
LOG_SYSERR, /* problem with system ressources: bad file, no mem */
LOG_IOERR, /* problem on IO */
LOG_ERROR, /* unusable input, database or query */
LOG_WARN, /* bad input */
LOG_INFO, /* some major event like opening a db */
LOG_VERBOSE, /* any event like reading a record */
LOG_TRACE, /* database content (log_rec) */
/* highest levels only with debug built */
LOG_DEBUG, /* lots of processing details */
LOG_ALL /* just everything, even built-in content */
} log_level;
extern log_level log_lev;
#define LOG_DO( lev ) ((lev) <= log_lev)
#define LOG_DONT( lev ) ((lev) > log_lev)
/**
print some log message to stderr or some configured logdevice.
if a non-positive code is given,
LOG_FATAL is used for -LERR_TRASH and below,
LOG_SYSERR is used as level for LERR_IO and below,
LOG_ERROR else.
if a positive code is given, this is the loglevel, and code is
-1 for LOG_FATAL or LOG_ERROR,
derived from errno for LOG_SYSERR,
0 else.
@param code errorcode (-LERR_xxx) or loglevel as described above
@return a non-positive code as described above
*/
extern int log_msg ( int code, const char *fmt, ... );
#ifndef NDEBUG
# define LOG_DBG log_msg
#else
# ifdef __GNUC__
# define LOG_DBG( args... )
# else
# define LOG_DBG (void) /* compiler should dispose statement off */
# endif
#endif
/* content logging is usually on level LOG_TRACE */
#define LOG_REC( rec, desc ) log_rec( LOG_TRACE, rec, desc )
#define LOG_HEX( mem, len ) log_hex( LOG_TRACE, mem, len )
/**
dump a record
@param level loglevel necessary for output to occurr.
*/
extern void log_rec ( log_level level, long *rec, const char **desc );
/**
hexdump memory
@param level loglevel necessary for output to occurr.
*/
extern void log_hex ( log_level level, const void *mem, int len );
/**
do a malloc and print error if out of memory.
*/
extern void *lmalloc ( long size );
/* more specific log functions in log.h */
#define LUTI_H
#endif /* LUTI_H */
./openisis.c 0100644 0001130 0001130 00000035555 07550303300 011675 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
/*
$Id: openisis.c,v 1.29 2002/10/07 13:06:08 kripke Exp $
main file of openisis executable.
*/
#ifdef HAVE_PTHREAD
#define HAVE_THREADS
#endif
#include
#include /* NULL */
#include /* printf */
#include /* free */
#include /* strcmp */
#include /* isprint */
#ifdef WIN32
# include
# include
# define timeval _timeb
#else
# include /* gettimeofday */
#endif
#include /* gettimeofday */
#ifdef HAVE_PTHREAD
#include /* threaded crashtest */
#endif
#include "openisis.h"
#include "lio.h"
/* ************************************************************
private types
*/
typedef enum { /* what to do */
DO_DUMP,
DO_SCAN, /* simple full scan */
DO_SEARCH, /* basic index based searching */
DO_TERMS, /* list terms */
DO_CHECK, /* check db */
DO_PERF, /* do random reads for performance checking */
DO_CRASH, /* do multi-threaded crashtest */
DO_SPLIT, /* split a field value */
DO_STREAM, /* stream in records */
DO_MFNLIST, /* fetch records by mfn list */
DO_VUTF /* validate UTF-8 input */
} todo;
typedef enum {
FMT_MFN, /* rowid only */
FMT_MFNF, /* rowid, 1st field */
FMT_PROP, /* property style */
FMT_ZML /* ZML style */
} format;
/* ************************************************************
private data
*/
static const char *pft;
static const char **term, **val;
static OpenIsisSet *post;
long nterm, nval;
static int db = -1, wdb = -1;
/* ************************************************************
private functions
*/
static int argchk ( const char *param, const char *n, const char *v )
{
return strcmp( param, n ) ? 0 : NULL != v ? 1 :
(fprintf( stderr, "no value for param '%s'", param ), 0);
} /* argchk */
static long print ( OpenIsisRec *r, int freeit, format f )
{
union { OpenIsisRec r; char buf[10000]; } x;
int i;
if ( NULL != r ) {
if ( NULL != pft ) {
OpenIsisRec *q;
OPENISIS_INITBUF(x);
q = openIsisFmt( &x.r, pft, r );
if ( freeit )
free( r );
freeit = q != &x.r;
r = q;
}
if ( FMT_MFN == f )
printf( "%ld\n", r->rowid );
else if ( FMT_MFNF == f )
printf( "%ld %.*s\n", r->rowid,
0 == r->len ? 1 : (int)r->field[0].len,
0 == r->len ? "-" : r->field[0].val );
else if ( FMT_ZML == f ) {
printf( "0\t%ld\n", r->rowid );
for ( i=0; ilen; i++ ) {
if ( r->field[i].val )
printf( "%ld\t%.*s\n", r->field[i].tag,
(int)r->field[i].len, r->field[i].val );
else
printf( "%ld\t%ld\n", r->field[i].tag, r->field[i].len );
}
printf( "\014\n" ); /* ^L = FormFeed */
} else for ( i=0; ilen; i++ ) {
if ( ! r->field[i].val ) { /* shouldn't happen -- numeric ? */
printf( "%ld.?=%ld\n", r->rowid, r->field[i].len );
continue;
}
printf( "%ld.%ld=%.*s\n", r->rowid, r->field[i].tag,
(int)r->field[i].len, r->field[i].val );
if ( r->field[i].len && '^' == *r->field[i].val ) { /* split subfields */
OpenIsisRec *rf = openIsisReadField( NULL, r->field+i );
if ( NULL != rf ) {
int j;
for ( j=0; jlen; j++ )
printf( "%ld.%ld.%c=%.*s\n", r->rowid, r->field[i].tag,
isprint( (int)rf->field[j].tag ) ?
(int)rf->field[j].tag : ' ',
(int)rf->field[j].len, rf->field[j].val );
free( rf );
}
}
}
if ( 0 <= wdb ) {
int ok = openIsisWrite( wdb, r );
printf( "wrote mfn %ld (%d)\n", r->rowid, ok );
}
if ( freeit )
free( r );
}
fflush( stderr );
fflush( stdout );
return NULL == r ? -1 : r->rowid;
} /* print */
static void printid ( long id, format f )
{
if ( FMT_MFN == f && 0 > wdb )
printf( "%ld\n", id );
else
print( openIsisReadRow( db, id ), !0, f );
} /* printid */
/* timing utility. set the timeval, return milliseconds since last call. */
#ifdef WIN32
static long millis ( struct _timeb *tb )
{
struct _timeb otb = *tb;
_ftime( tb );
return (tb->time - otb.time)*1000 + (tb->millitm - otb.millitm);
} /* millis */
#else
static long millis ( struct timeval *tv )
{
struct timeval otv = *tv;
gettimeofday( tv, NULL );
return (tv->tv_sec - otv.tv_sec)*1000 + (tv->tv_usec - otv.tv_usec)/1000;
} /* millis */
#endif
#ifdef HAVE_PTHREAD
int myOpenIsisLockFunc ( int lock )
{
static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER; /* the "fast" kind */
return lock ? pthread_mutex_lock( &mut ) : pthread_mutex_unlock( &mut );
}
#endif
/* (p)thread routine */
static void * run ( void *arg )
{
struct timeval tv;
long m;
int start = *(int *)arg;
int i;
millis( &tv );
for ( i=start+1; i++!=start; ) {
int j;
if ( i >= nterm )
i = 0;
for ( j=0; j= r->len )
fprintf( stderr, "no fields for %ld\n", mfn );
else if ( r->field[0].len != (long)strlen(val[mfn])
|| memcmp( r->field[0].val, val[mfn], r->field[0].len )
)
fprintf( stderr, "mismatch on %ld\n", mfn );
free( r );
}
}
m = millis(&tv);
printf( "thread %d terminated after %.3f seconds\n",
start, m/1000. );
return (void *)m;
} /* run */
/* multithreaded crashtest */
static int crash ( const char *pre )
{
struct timeval tv;
union { OpenIsisRec r; char buf[10000]; } x;
long l;
int p = 0;
int i, pass;
millis( &tv );
x.r.len = 0;
x.r.bytes = sizeof(x);
nterm = 0;
while ( NULL != openIsisTerm( &x.r, db, pre ) && x.r.len )
nterm += x.r.len;
printf( "%ld terms\n", nterm );
printf( "%.3f sec\n", millis(&tv)/1000. );
term = (const char **)malloc( nterm * sizeof(*term) );
post = (OpenIsisSet *)malloc( nterm * sizeof(*post) );
nterm = 0;
nval = 0;
while ( NULL != openIsisTerm( &x.r, db, pre ) && x.r.len ) {
for ( i=0; ilen = 0;
openIsisQuery( set, db, c, OPENISIS_QRY_KEYEQ, 0 );
if ( 0 >= set->len ) {
fprintf( stderr, "no results for '%s'\n", c );
return 1;
}
for ( cnt = 0; cnt < set->len; cnt++ )
if ( nval < set->id[cnt] )
nval = set->id[cnt];
p += set->len;
}
}
printf( "%d postings max mfn %ld\n", p, nval );
printf( "%.3f sec\n", millis(&tv)/1000. );
val = (const char **)malloc( (1+nval) * sizeof(*val) );
for ( i=1; i<=nval; i++ ) {
OpenIsisRec *r = openIsisReadRow( db, i );
val[i] = NULL;
if ( NULL == r ) {
fprintf( stderr, "no rec %d\n", i );
continue;
}
if ( 0 < r->len ) {
char *c = malloc( r->field[0].len + 1 );
memcpy( c, r->field[0].val, r->field[0].len );
c[ r->field[0].len ] = 0;
val[i] = c;
}
free( r );
}
l = millis(&tv);
printf( "sequential read %ld rows in %.3f seconds %ld rows per sec\n",
nval, l/1000., nval*1000/l );
i = 0;
run( &i );
l = millis(&tv);
run( &i );
l = millis(&tv);
run( &i );
l = millis(&tv);
printf( "in-thread run in %.3f seconds %ld rows per sec\n",
l/1000., p*1000/l );
#ifdef HAVE_THREADS
#define passes 3
{
static int nThreads[] = { 8, 2, 1, 4 };
int res[ sizeof(nThreads)/sizeof(nThreads[0]) ];
pthread_t th[ 8 /* max. nThreads */ ];
int start[ sizeof(th)/sizeof(th[0]) ];
for ( i=0; i<(int)(sizeof(nThreads)/sizeof(nThreads[0])); i++ )
res[i] = 0;
for ( pass=0; passvalue\n"
);
return 1;
}
if ( NULL != dowrite && 0 > (wdb = openIsisOpen( dowrite, NULL, 0 )) ) {
fprintf( stderr, "could not open write target db '%s'\n", dowrite );
return 2;
}
switch ( what ) {
case DO_DUMP: {
long max = openIsisMaxRowid( db );
long rowid;
for ( rowid = 1; rowid <= max; rowid++ )
printid( rowid, fmt );
} break; /* DO_DUMP */
case DO_MFNLIST: {
char buf[32];
while ( NULL != fgets( buf, sizeof(buf), stdin ) )
printid( atol(buf), fmt );
} break; /* DO_MFNLIST */
case DO_SCAN: {
long max = openIsisMaxRowid( db );
long rowid;
for ( rowid = 1; 0 < rowid && rowid <= max; rowid++ )
rowid = print( openIsisScan( db, rowid, 0, search ), !0, fmt );
} break; /* DO_SCAN */
case DO_SEARCH: {
long cnt;
OpenIsisSet set;
set.len = 0;
openIsisQuery( &set, db, search, searchmode, 0 );
if ( 0 >= set.len ) {
fprintf( stderr, "no results for '%s'\n", search );
return 1;
}
fprintf( stderr, "%ld\trows for\t%s\n", set.len, search );
for ( cnt = 0; cnt < set.len; cnt++ )
printid( set.id[cnt], fmt );
} break; /* DO_SEARCH */
case DO_TERMS: {
union { OpenIsisRec r; char buf[10000]; } x;
x.r.len = 0;
x.r.bytes = sizeof(x);
while ( NULL != openIsisTerm( &x.r, db, search ) && x.r.len ) {
/* printf( "%ld terms\n", x.r.len ); */
for ( i=0; ilen; i++ )
printf( "%c=%.*s\n", (int)r->field[i].tag,
(int)r->field[i].len, r->field[i].val );
} break;
case DO_STREAM: {
LioStream ios = { "stdin", 0, LIO_IN, 0, 0, 0, "" };
OpenIsisRecStream rs = { NULL, 0, NULL, NULL, NULL };
rs.in = &ios; /* some gcc versions need it this way */
while ( 0 < (i = openIsisReadStream( &rs )) )
print( rs.rec, 0, fmt );
} break;
case DO_VUTF: {
char buf[1024];
int t = 0, f = 0, g;
while ( 0 < (g = fread( buf, 1, sizeof(buf), stdin )) ) {
int l = openIsisValidUTF8( buf, g, &f );
if ( l ) {
fprintf( stderr, "at total %d = %d+%d\n", l-1+t, l-1, t );
return 1;
}
t += g;
}
}
} /* switch ( what ) */
if ( 0 <= db )
openIsisClose( db );
if ( 0 <= wdb )
openIsisClose( wdb );
return 0;
} /* openisis main */
./openisis.h 0100644 0001130 0001130 00000040560 07547355244 011716 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
#ifndef OPENISIS_H
/*
$Id: openisis.h,v 1.24 2002/10/04 18:14:40 kripke Exp $
public interface of the openisis lib.
*/
#ifndef __STDC__
# ifndef const
# define const
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
/* ************************************************************
system initialization
*/
/** a function to provide a global lock in a multi threaded process.
@param lock if != 0, then lock, else unlock
@return 0 on success
sample:
int myOpenIsisLockFunc ( int lock )
{
static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
return lock ? pthread_mutex_lock( &mut ) : pthread_mutex_unlock( &mut );
}
*/
typedef int OpenIsisLockFunc ( int lock );
/** initialize system with some common parameters and a lock func.
@param lock a function to provide a global lock in a multi threaded process.
*/
extern int openIsisInit ( int argc, const char **argv,
OpenIsisLockFunc lock );
/* ************************************************************
db access
*/
/**
open an isis database.
The following options are supported as text-style parameters:
db <name>
basename of database. same as param dbname.
dbpath <path>
path to database. usefull when opening secondary indexes
or other files whose names are not based on the db basename.
v <level>
set verbosity level
@param dbname basename of database.
the dbhome arg, if given, will be prepended, and the ISIS file
extensions appended to build the actual filename.
Therefore, depending on the value of dbhome and location
of files, this may need to include a path.
db may be NULL, if argv includes a dbname arg.
@param argv array of names and values.
May be NULL.
argv contains parameter names, which may be prefixed
by a dash '-'. Depending on the actual parameter, the next
string may or must contain a corresponding parameter value.
An optional parameter value may be omitted, if at end of
argv or the next name is prefixed with a dash.
To avoid ambiguity, values never start with a dash.
See above for a list of supported parameter names.
Unrecognized parameter names are ignored.
@param argc length of argv.
You may supply -1 to indicate that argv is NULL terminated.
@return some non-negative dbid on success,
some negative errorcode on error.
Errorcodes include -ENOENT, -ENOMEM, -EACCESS and others
returned by file access functions.
*/
extern int openIsisOpen ( const char *dbname, const char **argv, int argc );
enum {
OPENISIS_CHK_ONLY = 0x01, /* check only, do not fix */
OPENISIS_CHK_XRF = 0x02, /* check the xrf */
OPENISIS_CHK_FIX = 0xfe, /* check all and fix */
OPENISIS_CHK_ALL = 0xff /* check all, no fix */
};
/** check and optionally fix the database.
*/
extern int openIsisCheck ( int dbid, int flags );
/**
close a database.
@param dbid id returned by openIsisOpen
@return 0 on success, some negative errorcode else.
*/
extern int openIsisClose ( int dbid );
/**
get highest rowid in db
@param dbid id returned by openIsisOpen
@return positive rowid on success, 0 if unknown, negative on error
*/
extern long openIsisMaxRowid ( int dbid );
/* ************************************************************
record access
*/
/** raw version of an ISIS field.
* off gives an offset from the start of the record taken as char*.
* values are *NOT* terminated by a 0 byte.
*/
typedef struct {
/*
although isis-1 dbs have only 2 bytes for field tags (numbers),
we use longs with the lower 3 bytes for the tag.
The highest byte may have special usage in some context.
*/
long tag;
long off;
long len;
} OpenIsisFld;
/** cooked version of an ISIS field.
* values are *NOT* terminated by a 0 byte.
*/
typedef struct {
long tag;
const char *val;
long len;
} OpenIsisField;
/** raw version of an ISIS Record.
*/
typedef struct {
long dbid;
long rowid;
long bytes; /* total avail bytes */
long used; /* total used bytes (originally BWB) */
long fields; /* avail number of fields (originally BWP) */
long base; /* byte offset of contents area (after fields) */
long len; /* used number of fields */
long state;
OpenIsisFld field[1];
} OpenIsisRaw;
/* compute base from fields (for both raw and cooked record) */
#define OPENISIS_BASESZ( nfields ) ((long)sizeof(long)*(8 + 3*(nfields)))
/* initialise record of f fields, n bytes */
#define OPENISIS_INITREC( r, n, f ) \
do { \
memset( r, 0, n ); \
(r)->bytes = (n); \
(r)->fields = (f); \
(r)->used = (r)->base = OPENISIS_BASESZ( (r)->fields ); \
} while (0)
/* init with std 1 field per 100 bytes */
#define OPENISIS_INITSTD( r, n ) OPENISIS_INITREC( r, n, (n)/100 )
/* init a buffer structure which has a member r */
#define OPENISIS_INITBUF( x ) OPENISIS_INITSTD( &x.r, sizeof(x) )
/* check some invariants (for both raw and cooked record) */
#define OPENISIS_RECOK( r ) \
( (r)->base <= (r)->used \
&& (r)->used <= (r)->bytes \
&& (r)->len <= (r)->fields \
&& (r)->base == OPENISIS_BASESZ( (r)->fields ) )
/** cooked version of an ISIS Record.
*/
typedef struct {
long dbid;
long rowid;
long bytes; /* total avail bytes */
long used; /* total used bytes (originally BWB) */
long fields; /* avail number of fields (originally BWP) */
long base; /* byte offset of contents area (after fields) */
long len; /* used number of fields */
long state;
OpenIsisField field[1];
} OpenIsisRec;
/**
clone record or create empty record.
with room >= 0 and rec != NULL,
make sure that there is at least one field and room bytes available.
else, it works in shrink-to-fit mode: exactly the required bytes are used.
@param rec to extend, may be NULL
@param room number of required free bytes (<0 for shrink-to-fit)
@param discard wether the original record should be freed after extension
@return a rec with suitable room
*/
extern OpenIsisRec* openIsisClone ( OpenIsisRec *r, long room, int discard );
/**
printf a new field of up to 1024 bytes at the end of record.
*/
extern OpenIsisRec* openIsisPrintf ( OpenIsisRec *r, int discard,
int tag, const char *fmt, ... );
/**
check for free headspace in a record.
make sure that there is at least one field and room bytes available.
NOTE: this may change recp
@see openIsisClone
*/
#define OPENISIS_CHKFREE( recp, room, disc ) \
do { if ( NULL == (recp) \
|| (recp)->len >= (recp)->fields \
|| (recp)->bytes < (recp)->used + (room) \
) (recp) = openIsisClone( (recp), (room), (disc) ); } while (0)
/**
add a field to a record.
a new field entry and n bytes space are provided.
if the pointer s is not NULL, n bytes are copied from it to the field.
NOTE: this may change recp
@see openIsisClone
*/
#define OPENISIS_ADD( recp, ntag, s, n, disc ) \
do { \
long _ll = (n); \
OPENISIS_CHKFREE( recp, _ll, disc ); \
if ( NULL != (recp) ) { \
OpenIsisField *_ff = (recp)->field + (recp)->len++; \
_ff->tag = (ntag); \
_ff->val = ((char*)recp)+(recp)->used; \
if ( NULL == s ) \
_ff->len = 0; \
else \
memcpy( (char*)_ff->val, s, _ff->len = _ll ); \
(recp)->used += _ff->len; \
} \
} while(0)
/**
append n bytes to the last field of a record.
NOTE: this may change recp
@see openIsisClone
*/
#define OPENISIS_CAT( recp, s, n, disc ) \
do { \
OpenIsisField *_ff; \
long _ll = (n); \
assert ( (recp) ); \
assert ( (recp)->len ); \
if ( (recp)->bytes < (recp)->used + _ll ) \
(recp) = openIsisClone( (recp), _ll, (disc) ); \
_ff = (recp)->field + (recp)->len - 1; \
memcpy( (char*)(_ff->val+_ff->len), s, _ll ); \
_ff->len += _ll; \
(recp)->used += _ll; \
} while(0)
/* like OPENISIS_ADD, but based on strlen */
#define OPENISIS_SADD( recp, ntag, s, disc ) \
OPENISIS_ADD( recp, ntag, s, strlen(s), disc )
/* like OPENISIS_CAT, but based on strlen */
#define OPENISIS_SCAT( recp, s, disc ) \
OPENISIS_CAT( recp, s, strlen(s), disc )
/**
read a cooked row.
free the returned pointer after usage.
@param db id returned by openIsisOpen
@param rowid the mfn
@return an OpenIsisRec or NULL, if row not available.
*/
extern OpenIsisRec *openIsisReadRow ( int db, long rowid );
/** read raw. */
extern OpenIsisRaw *openIsisReadRaw ( int db, long rowid );
/**
simple full scan search.
read the first row with rowid >= given rowid
and containing txt in subfield tag (any, if < 0).
free the returned pointer after usage.
@param db id returned by openIsisOpen
@param rowid the mfn
@param tag field in which to look for key, 0 for any
@return an OpenIsisRec or NULL, if row not available.
*/
extern OpenIsisRec *openIsisScan (
int db, long rowid, long tag, const char *key );
/**
create a record from a field, i.e. split subfields.
if a rec buffer is given, it is filled up to it's rec->len and returned.
else an appropriately allocated rec is returned,
which must be freed after usage.
NOTE: USE WITH CARE, IT SOMEWHAT BREAKS THE GENERAL CONTRACT OF REC!
DO NOT TRY TO EXTEND THE RETURNED RECORD !!!
The rec does NOT have it's own data buffer,
but instead field values are pointers into the input field's value.
The returned rec will therefore become invalid,
if the inpur field is freed.
@param rec a rec buffer to fill or NULL
@param field some string to be split up at '^' characters
@return an OpenIsisRec or NULL, if splitting failed.
*/
extern OpenIsisRec *openIsisReadField ( OpenIsisRec *rec,
const OpenIsisField* field );
/**
get a term for key w/ or w/o the postings.
prefix mode is determined by a trailing '$' in key.
for prefix mode, the last preset field in the record
gives a starting point, i.e. terms up to and including
it are ignored.
thus, if there are more matching terms than fit in the record,
the search can be continued by passing the same record.
@return an OpenIsisRec or NULL
*/
extern OpenIsisRec* openIsisTerm ( OpenIsisRec *rec,
int db, const char *key );
#ifndef OPENISIS_SETLEN
#define OPENISIS_SETLEN 1000
#endif
/**
record set type as returned from openIsisSearch.
first entry is the set length n, followed by rowids 1..n.
actual array length is thus n+1.
*/
typedef struct {
long len;
long id[OPENISIS_SETLEN];
} OpenIsisSet;
/* query mode */
enum {
/* search given string using index */
OPENISIS_QRY_KEYEQ, /* index scan key equal */
OPENISIS_QRY_KEYPF, /* index scan key prefix */
OPENISIS_QRY_KEYAT, /* index scan key auto (checks for '$') */
/* fulltext scan for given string */
OPENISIS_QRY_SCANE=64, /* fulltext scan for equal */
OPENISIS_QRY_SCANC, /* fulltext scan for contains */
/* parse string as query expression */
OPENISIS_QRY_SIMPLE=128, /* simple left-to-right binding */
OPENISIS_QRY_PROPER /* proper binding using precedence, () */
};
/**
query the db.
@param set set buffer to fill.
set->len must contain the actual buffer length (# of ids).
0 means default length OPENISIS_SETLEN.
@param db id returned by openIsisOpen
@param key keyword or query expression to search for
@param mode query mode, optionally |ed with a field tag<<16.
a field tag affects key and scan modes.
@param skip mfns < skip are ignored, useful to continue a query
@return the number of rowids in set (= set->len) on success, <0 on error.
*/
extern long openIsisQuery ( OpenIsisSet *set, int db,
const char *key, long mode, long skip );
/* ************************************************************
utilities
*/
/**
set logging level and device.
@param level the logging level
may be given numerical or as ascii char in -ofsewivtda for
nochange(-1), off (0), fatal (1), syserr, error, warn, info, verbose,
trace, debug, all(9)
default is set to error
@param filename specifies, if not NULL, a file to append log to
*/
extern void openIsisLog ( int level, const char *filename );
/**
convert string to html by replacing the four characters
<,>,& and ".
free the returned pointer after usage.
XML has a fifth standard entity apos,
but many HTML clients don't grok it.
*/
extern char *openIsis2Html ( const char *str, long len );
extern int openIsisValidUTF8 ( void *mem, int len, int *tof );
enum {
OPENISIS_FMT_SHARP = 32, /* # */
OPENISIS_FMT_SLASH, /* / */
OPENISIS_FMT_PERCENT, /* % */
OPENISIS_FMT_OPEN, /* { */
OPENISIS_FMT_CLOSE, /* } */
OPENISIS_FMT_ESC, /* ! */
OPENISIS_FMT_B, /* bold */
OPENISIS_FMT_BOX, /* draw a box (0i) */
OPENISIS_FMT_BPICT, /* draw background pict (1sn) */
OPENISIS_FMT_C, /* column (1i) */
OPENISIS_FMT_CL, /* colour (1i) */
OPENISIS_FMT_COLS, /* colour table (s_) - emits a string of hex vals */
OPENISIS_FMT_FONTS, /* font table (s_) */
OPENISIS_FMT_FS, /* font size (1i) */
OPENISIS_FMT_F, /* font (1i) */
OPENISIS_FMT_I, /* italic */
OPENISIS_FMT_LINK, /* link (2s_) */
OPENISIS_FMT_LW, /* line width (1i) */
OPENISIS_FMT_M, /* indentation in twips(1in) */
OPENISIS_FMT_NC, /* new column (0i) */
OPENISIS_FMT_NEWLINE, /* set newline (1s) */
OPENISIS_FMT_NP, /* newpage (0i) */
OPENISIS_FMT_PICT, /* draw pict (1s) */
OPENISIS_FMT_QC, /* center */
OPENISIS_FMT_QJ, /* justify */
OPENISIS_FMT_TAB, /* tab (0i) */
OPENISIS_FMT_UL, /* underline */
OPENISIS_FMT_X /* blanks (1i) */
};
extern OpenIsisRec* openIsisFmt ( OpenIsisRec *buf,
const char *fmt, OpenIsisRec *r );
/**
for convenience: read and format in one.
*/
extern OpenIsisRec *openIsisRead ( OpenIsisRec *buf,
const char *fmt, int db, long rowid );
typedef struct _LioStream *OpenIsisStream;
typedef struct _OpenIsisRecStream {
OpenIsisStream in; /* input stream to read from */
int flg; /* flags */
OpenIsisRec *buf; /* original buffer */
OpenIsisRec *rec; /* record read */
OpenIsisRec *dict; /* dictionary of field names */
} OpenIsisRecStream;
/*
stream in records in some free-style format suitable for
mails, properties, syspar ...
fields are delimited by CR or NL or CR-NL
within a field, tag stops at blank,TAB,= or :
after a TAB, val starts immediately
after a = or :, val starts at 1st character not a blank
after a blank, val starts at 1st char not one of the above separators
processing
- a field or record separator is recognized anywhere
- within each field, the tag is read up to the first separator
(any "special" char, all controls, whitespace and most punctuation).
upon reaching a separator, the tag is identified.
if it is empty, and the record has a field, now new field is created.
else, if the tag cannot be identified,
a tag of -1 is used and the tag string appended as value.
- the separator consists of any number of leading blanks,
some separator char, and, unless the sepchar is TAB, trailing blanks.
if the current field is not empty, the separator is appended as value.
- the rest of the field is appended as value
- a field with empty tag and value (blank line)
terminates the record in most modes
- a field with empty tag (continuation line)
has it's separator AND value appended to the previous field
- if either a tag or a separator extend over more than 4k,
the stream is considered over limit.
*/
enum {
OPENISIS_STOPONEMPTY = 0x10,
OPENISIS_AUTOCLONE = 0x20
};
extern int openIsisReadStream ( OpenIsisRecStream *stream );
/**
write a record.
If the rowid is 0, the record will be modified by assigning a new rowid.
@param db id returned by openIsisOpen
@param rec the OpenIsisRec to write
@return 0 or an error code
*/
extern int openIsisWrite ( int dbid, OpenIsisRec *rec );
#ifdef __cplusplus
}
#endif /* extern "C" */
#define OPENISIS_H
#endif /* OPENISIS_H */
./openjsis.c 0100644 0001130 0001130 00000013472 07447676140 011714 0 ustar krip krip /*
openisis - an open implementation of the ISIS database
Version 0.8.x (microversion see file Version)
Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This software is dedicated to the memory of Eckart Dietrich.
This software is inspired by (but contains no code of) the iAPI
Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
See iAPI.txt for what it contains.
EOH */
/*
$Id: openjsis.c,v 1.10 2002/03/25 19:36:44 kripke Exp $
main file of openjsis java native lib.
*/
#include
#include /* NULL */
#include /* printf */
#include /* free */
#include /* isprint */
#include "openisis.h"
#include "org/openisis/NativeDb.h"
/* ************************************************************
private types
*/
/* ************************************************************
private data
*/
/* ************************************************************
private functions
*/
static jobjectArray rec2bytes (JNIEnv * env, OpenIsisRec *r )
{
jobjectArray jarr = NULL;
jintArray ia;
jint ji;
jbyteArray ba;
int i;
if ( NULL == r )
return NULL;
jarr = (*env)->NewObjectArray( env, 1+r->len,
(*env)->FindClass( env, "java/lang/Object" ), NULL );
if ( NULL == jarr )
return NULL;
ia = (*env)->NewIntArray( env, 1+r->len );
(*env)->SetObjectArrayElement( env, jarr, 0, ia );
ji = r->rowid;
(*env)->SetIntArrayRegion( env, ia, 0, 1, &ji );
for ( i=0; ilen; i++ ) {
OpenIsisField *f = &r->field[i];
ba = (*env)->NewByteArray( env, f->len );
if ( NULL == ba )
break;
ji = f->tag;
(*env)->SetIntArrayRegion( env, ia, i+1, 1, &ji );
(*env)->SetByteArrayRegion( env, ba, 0, f->len, (jbyte*)f->val );
(*env)->SetObjectArrayElement( env, jarr, i+1, ba );
}
return jarr;
} /* rec2bytes */
/* ************************************************************
package functions
*/
/* ************************************************************
public functions
*/
JNIEXPORT jshort JNICALL Java_org_openisis_NativeDb_nopen
(JNIEnv *env, jclass cls, jstring jdbname, jobjectArray jargs)
{
int db;
const char *dbname = NULL;
const char **argv = NULL;
int argc = 0;
jsize i;
(void)cls;
dbname = NULL==jdbname ? NULL :
(*env)->GetStringUTFChars( env, jdbname, NULL );
if ( NULL != jargs ) {
argc = (*env)->GetArrayLength( env, jargs );
argv = malloc( argc * sizeof(*argv) );
if ( NULL != argv )
for ( i=0; iGetStringUTFChars( env,
(*env)->GetObjectArrayElement( env, jargs, i ), NULL );
}
db = openIsisOpen( dbname, argv, argc );
fprintf( stderr, "org.openisis.Db::open '%s'\n",
NULL==dbname ? "" : dbname );
if ( NULL != dbname )
(*env)->ReleaseStringUTFChars( env, jdbname, dbname );
if ( NULL != argv )
for ( i=0; iReleaseStringUTFChars( env,
(*env)->GetObjectArrayElement( env, jargs, i ), argv[i] );
return db;
} /* Java_org_openisis_Db_open */
JNIEXPORT jobjectArray JNICALL Java_org_openisis_NativeDb_nreadRow
(JNIEnv * env, jclass cls, jshort db, jint rowid, jint tag, jbyteArray txt )
{
OpenIsisRec *r;
jobjectArray jarr = NULL;
(void)cls;
if ( NULL == txt )
r = openIsisReadRow( db, rowid );
else {
jsize l = (*env)->GetArrayLength( env, txt );
char *search = malloc( l+1 );
(*env)->GetByteArrayRegion( env, txt, 0, l, (jbyte*)search );
search[l] = 0;
r = openIsisScan( db, rowid, tag, search );
free( search );
}
jarr = rec2bytes( env, r );
if ( NULL != r ) free( r );
#ifndef NDEBUG
fflush( stderr );
#endif
return jarr;
} /* Java_org_openisis_Db_readRow */
JNIEXPORT jobjectArray JNICALL Java_org_openisis_NativeDb_nTerms
(JNIEnv * env, jclass cls, jshort db, jbyteArray term, jbyteArray start )
{
OpenIsisRec *r;
jobjectArray jarr = NULL;
jsize lterm = (*env)->GetArrayLength( env, term );
char *cterm = malloc( lterm+1 );
union { OpenIsisRec r; char buf[4096]; } x;
x.r.len = 0;
x.r.bytes = sizeof(x);
(void)cls;
(*env)->GetByteArrayRegion( env, term, 0, lterm, (jbyte*)cterm );
cterm[lterm] = 0;
if ( NULL != start ) {
char *cstart;
jsize lstart = (*env)->GetArrayLength( env, start );
if ( 128 < lstart )
lstart = 128;
cstart = x.buf + sizeof(x.r);
(*env)->GetByteArrayRegion( env, start, 0, lstart, (jbyte*)cstart );
x.r.field[0].val = cstart;
x.r.field[0].len = lstart;
x.r.len = 1;
}
r = openIsisTerm( &x.r, db, cterm );
free( cterm );
jarr = rec2bytes( env, r );
#ifndef NDEBUG
fflush( stderr );
#endif
return jarr;
} /* Java_org_openisis_Db_Terms */
JNIEXPORT jintArray JNICALL Java_org_openisis_NativeDb_nsearch
(JNIEnv * env, jclass cls, jshort db, jbyteArray key, jint mode)
{
jsize l;
char *search;
OpenIsisSet set;
jintArray ia;
(void)cls;
if ( NULL == key )
return NULL;
l = (*env)->GetArrayLength( env, key );
search = malloc( l+1 );
(*env)->GetByteArrayRegion( env, key, 0, l, (jbyte*)search );
search[l] = 0;
set.len = 0;
openIsisQuery( &set, db, search, mode, 0 );
free( search );
if ( 0 >= set.len )
return NULL;
ia = (*env)->NewIntArray( env, set.len );
(*env)->SetIntArrayRegion( env, ia, 0, set.len, set.id );
return ia;
} /* Java_org_openisis_Db_search */
./test.pl 0100755 0001130 0001130 00000006367 07505604210 011222 0 ustar krip krip #
#/*
# openisis - an open implementation of the ISIS database
# Version 0.8.x (microversion see file Version)
# Copyright (C) 2001-2002 by Erik Grziwotz, erik@openisis.org
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# This software is dedicated to the memory of Eckart Dietrich.
#
# This software is inspired by (but contains no code of) the iAPI
# Copyright (C) 2000 by Robert Janusz, rj@jezuici.krakow.pl.
# See iAPI.txt for what it contains.
#EOH */
#
# Before `make install' is performed this script should be runnable with
# `make test'. After `make install' it should work as `perl test.pl'
######################### We start with some black magic to print on failure.
# Change 1..1 below to 1..last_test_to_print .
# (It may become useful if the test is moved to ./t subdirectory.)
BEGIN { $| = 1; print "1..6\n"; }
END {print "not ok 1\n" unless $loaded;}
use OpenIsis;
$loaded = 1;
print "ok 1\n";
######################### End of black magic.
# Insert your test code below (better if it prints "ok 13"
# (correspondingly "not ok 13") depending on the success of chunk 13
# of the test code):
sub pok { print $_[1] ? '' : 'not ', 'ok ', $_[0], ' (', $_[2], ") \n"; }
OpenIsis::log(0,'');
pok( 2, 1, 'unfortunately the effect of logging cannot be tested here' );
$db = OpenIsis::open( "cds", "-dbpath", "../db/cds" );
pok( 3, $db >= 0, $db );
$maxrow = OpenIsis::maxRowid( $db );
pok( 4, $maxrow > 0, $maxrow );
$row = OpenIsis::read( $db, 28 );
for (keys(%$row)) {
next if 'mfn' eq $_;
$a = $row->{$_};
OpenIsis::MHL( @$a );
print STDERR "\t$_=",join( "\n\t", @$a ),"\n";
}
pok( 5, "HASH" eq ref($row), $row );
$row = OpenIsis::subfields( '^ap. 233-238 ^billus.' );
for (keys(%$row)) { print STDERR "\t$_=$row->{$_}\n"; }
pok( 6, "HASH" eq ref($row), $row );
@res = OpenIsis::query( $db, 'plant * water' );
print STDERR "mfns for 'plant * water': ", join(',',@res), "\n";
pok( 7, 2 == @res, scalar(@res) );
@res = OpenIsis::query( $db, 'devel$' );
print STDERR "mfns for 'development': ", join(',',@res), "\n";
pok( 8, 7 == @res, scalar(@res) );
for (@res) {
$row = OpenIsis::read( $db, $_ );
$title = $row->{'24'};
OpenIsis::MHL( @$title );
print STDERR "mfn ", $row->{'mfn'}, " title: ", $title->[0], "\n";
}
@res = OpenIsis::terms( $db, 'a$' );
print STDERR 'terms for "a$": ', join('; ',@res), "\n";
$str = OpenIsis::2html( 'lt: < gt: > amp: & quot: "' );
print STDERR "\t$str\n";
@res = OpenIsis::query( $db, 'Dacca' );
for (@res) {
$row = OpenIsis::read( $db, $_, 'V24, V26, V26^*[..], V71,' );
for (sort(keys(%$row))) {
$vals = $row->{$_};
for $v (@$vals) { print STDERR "\t$_=$v\n"; }
}
}
./testres.txt 0100644 0001130 0001130 00000107542 07425516376 012152 0 ustar krip krip 47.24=Hydrological achievements and social problems
47.26=^c1966
47.26.c=1966
47.30=^ap. 357-359
47.30.a=p. 357-359
47.44=Scientific problems of the humid tropical zone deltas and their implications: proceedings of the Dacca Symposium
47.69=Paper on:
47.70=Versluys, J.D.N.
50.24=Influence of man's traditional activities on soils and vegetation
50.26=^c1966
50.26.c=1966
50.30=^ap. 301-304
50.30.a=p. 301-304
50.44=Scientific problems of the humid tropical zone deltas and their implications: proceedings of the Dacca Symposium
50.69=Paper on:
50.70=Boesch, H.
51.24=Deltas: a basis of culture and civilization
51.26=^c1966
51.26.c=1966
51.30=^ap. 295-300
51.30.a=p. 295-300
51.44=Scientific problems of the humid tropical zone deltas and their implications: proceedings of the Dacca Symposium
51.69=Paper on:
51.70=B<=ue>del, J.
71.24=Agriculture with special reference to rice cultivation in humid tropical zone deltas
71.26=^c1966
71.26.c=1966
71.30=^ap. 305-316
71.30.a=p. 305-316
71.44=Scientific problems of the Dacca Symposium
71.50=Incl. bibl.
71.69=Paper on:
71.70=Goor, G.A.W. van de
73.24=Need for land-use studies in East Pakistan
73.26=^c1966
73.26.c=1966
73.30=^ap. 329-332
73.30.a=p. 329-332
73.44=Scientific problems of the humid tropical zone deltas and their implications: proceedings of the Dacca Symposium
73.50=Incl. bibl.
73.69=Paper on:
73.70=Ahmad, Nazir
73.70=Khan, F. Karim
79.76=La Politique culturelle en Rpublique unie du Cameroun^zfre
79.44=Studies and documents on cultural policies
79.50=Incl. bibl.
79.69= -
79.24=Cultural policy in the United Republic of Cameroon
79.26=^aParis ^bUnesco Press^c1976
79.26.a=Paris
79.26.b=Unesco Press
79.26.c=1976
79.30=^a91 p.^billus.^c24 cm.
79.30.a=91 p.
79.30.b=illus.
79.30.c=24 cm.
79.70=Bahoken, J.C.
79.70=Atangana, Engelbert
86.24=Higher education and the labour market in Zambia: expectations and performance
86.26=^aParis^bUnesco Press^bIIEP^aLusaka^bUniversity of Zambia
86.26.a=Paris
86.26.b=Unesco Press
86.26.b=IIEP
86.26.a=Lusaka
86.26.b=University of Zambia
86.30=^a373 p.^billus., map^c24 cm.
86.30.a=373 p.
86.30.b=illus., map
86.30.c=24 cm.
86.50=Incl. bibl.
86.69= -
86.70=Sanyal, Bikas C.
86.70=Case, John H.
86.70=Dow, Philip S.
86.70=Jackman, Mary Elizabeth
86.71=University of Zambia
86.71=IIEP
87.24=Innovation in science education - world-wide
87.26=^aParis^bUnesco Press^c1976
87.26.a=Paris
87.26.b=Unesco Press
87.26.c=1976
87.30=^a249 p.^billus.^c24 cm.
87.30.a=249 p.
87.30.b=illus.
87.30.c=24 cm.
87.70=Baez, Albert V.
87.50=Incl. bibl.
87.69= -
87.76=L'Innovation dans l'enseignement des sciences: synthse mondiale^zfre
93.24=Broadcast training techniques
93.26=^aParis^bUnesco Press^c1976
93.26.a=Paris
93.26.b=Unesco Press
93.26.c=1976
93.30=^a122 p.^billus.^c22 cm.
93.30.a=122 p.
93.30.b=illus.
93.30.c=22 cm.
93.50=Incl. bibl.
93.69= -
93.70=Dance, F. Robert
93.76=Tcnicas de formacin del personal de comunicacin^zspa
100.24=Micropolitan development: theory and practice of greater-rural economic development
100.26=^aAmes^bIowa State University Press^c1976
100.26.a=Ames
100.26.b=Iowa State University Press
100.26.c=1976
100.30=^a456 p.^bmap
100.30.a=456 p.
100.30.b=map
100.50=Incl. bibl.
100.69= -
100.70=Tweeten, Luther G.
100.70=Brinkman, George Lorris
105.76=Les Statistiques de la science et de la technologie en Amrique latine ^zfre
105.76=Las Estadsticas de la ciencia y la tecnologa en Amrica Latina^zspa
105.24=Statistics on science and technology in Latin America; experience with Unesco pilot projects, 1972-1974
105.26=^aParis^bUnesco Press^c1976
105.26.a=Paris
105.26.b=Unesco Press
105.26.c=1976
105.30=^a76 p.^c27 cm.
105.30.a=76 p.
105.30.c=27 cm.
105.70=Thbaud, Schiller
105.44=Statistical reports and studies^v20
105.50=Incl. bibl.
105.69= -
105.72=Coloquio sobre los Aspectos Nacionales e Internacionales de los Sistemas de Datos para las Estadsticas relativas a la Ciencia y la Tecnologa en Amrica Latina^pCaracas^d1974
108.24=Comunicaao/incomunicaao no Brasil
108.26=^aSao Paulo ^bEdioes Loyola^c1976
108.26.a=Sao Paulo
108.26.b=Edioes Loyola
108.26.c=1976
108.30=^a207 p.^billus.
108.30.a=207 p.
108.30.b=illus.
108.50=(Biblioteca Universitria de Comunicaao, vol.1). Incl. bibl.
108.69= -
108.70=Melo, Jos Marques de
108.71=Uniao Crista Brasileira de Comunicaao Social
111.24=Multinational corporation and social change
111.26=^aNew York ^bPraeger Publishers^c1976
111.26.a=New York
111.26.b=Praeger Publishers
111.26.c=1976
111.30=^a234 p.
111.30.a=234 p.
111.50=Incl. bibl.
111.69= -
111.70=Apter, David Ernest
111.70=Goodman, Louis Wolf
114.24=Educational development; a new discipline for self-renewal
114.26=^aEugene, Or.^bCenter for the Advanced Study of Educational Administration^c1973
114.26.a=Eugene, Or.
114.26.b=Center for the Advanced Study of Educational Administration
114.26.c=1973
114.30=^a343 p.^billus.
114.30.a=343 p.
114.30.b=illus.
114.50=(CASEA monograph, no.24). Incl. bibl.
114.69= -
,
a service of merconic, Berlin, Germany.
As a student's site, allmaxx supports open software with a focus on
education and knowledge management.
See also the open community for science.