mirror of
https://github.com/pgsty/minio.git
synced 2026-08-09 07:43:29 +03:00
Refactor s3select to support parquet. (#7023)
Also handle pretty formatted JSON documents.
This commit is contained in:
Bala FA
+339
@@ -0,0 +1,339 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 2, June 1991
|
||||
|
||||
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The licenses for most software are designed to take away your
|
||||
freedom to share and change it. By contrast, the GNU General Public
|
||||
License is intended to guarantee your freedom to share and change free
|
||||
software--to make sure the software is free for all its users. This
|
||||
General Public License applies to most of the Free Software
|
||||
Foundation's software and to any other program whose authors commit to
|
||||
using it. (Some other Free Software Foundation software is covered by
|
||||
the GNU Lesser General Public License instead.) You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
this service if you wish), that you receive source code or can get it
|
||||
if you want it, that you can change the software or use pieces of it
|
||||
in new free programs; and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to make restrictions that forbid
|
||||
anyone to deny you these rights or to ask you to surrender the rights.
|
||||
These restrictions translate to certain responsibilities for you if you
|
||||
distribute copies of the software, or if you modify it.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must give the recipients all the rights that
|
||||
you have. You must make sure that they, too, receive or can get the
|
||||
source code. And you must show them these terms so they know their
|
||||
rights.
|
||||
|
||||
We protect your rights with two steps: (1) copyright the software, and
|
||||
(2) offer you this license which gives you legal permission to copy,
|
||||
distribute and/or modify the software.
|
||||
|
||||
Also, for each author's protection and ours, we want to make certain
|
||||
that everyone understands that there is no warranty for this free
|
||||
software. If the software is modified by someone else and passed on, we
|
||||
want its recipients to know that what they have is not the original, so
|
||||
that any problems introduced by others will not reflect on the original
|
||||
authors' reputations.
|
||||
|
||||
Finally, any free program is threatened constantly by software
|
||||
patents. We wish to avoid the danger that redistributors of a free
|
||||
program will individually obtain patent licenses, in effect making the
|
||||
program proprietary. To prevent this, we have made it clear that any
|
||||
patent must be licensed for everyone's free use or not licensed at all.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
||||
|
||||
0. This License applies to any program or other work which contains
|
||||
a notice placed by the copyright holder saying it may be distributed
|
||||
under the terms of this General Public License. The "Program", below,
|
||||
refers to any such program or work, and a "work based on the Program"
|
||||
means either the Program or any derivative work under copyright law:
|
||||
that is to say, a work containing the Program or a portion of it,
|
||||
either verbatim or with modifications and/or translated into another
|
||||
language. (Hereinafter, translation is included without limitation in
|
||||
the term "modification".) Each licensee is addressed as "you".
|
||||
|
||||
Activities other than copying, distribution and modification are not
|
||||
covered by this License; they are outside its scope. The act of
|
||||
running the Program is not restricted, and the output from the Program
|
||||
is covered only if its contents constitute a work based on the
|
||||
Program (independent of having been made by running the Program).
|
||||
Whether that is true depends on what the Program does.
|
||||
|
||||
1. You may copy and distribute verbatim copies of the Program's
|
||||
source code as you receive it, in any medium, provided that you
|
||||
conspicuously and appropriately publish on each copy an appropriate
|
||||
copyright notice and disclaimer of warranty; keep intact all the
|
||||
notices that refer to this License and to the absence of any warranty;
|
||||
and give any other recipients of the Program a copy of this License
|
||||
along with the Program.
|
||||
|
||||
You may charge a fee for the physical act of transferring a copy, and
|
||||
you may at your option offer warranty protection in exchange for a fee.
|
||||
|
||||
2. You may modify your copy or copies of the Program or any portion
|
||||
of it, thus forming a work based on the Program, and copy and
|
||||
distribute such modifications or work under the terms of Section 1
|
||||
above, provided that you also meet all of these conditions:
|
||||
|
||||
a) You must cause the modified files to carry prominent notices
|
||||
stating that you changed the files and the date of any change.
|
||||
|
||||
b) You must cause any work that you distribute or publish, that in
|
||||
whole or in part contains or is derived from the Program or any
|
||||
part thereof, to be licensed as a whole at no charge to all third
|
||||
parties under the terms of this License.
|
||||
|
||||
c) If the modified program normally reads commands interactively
|
||||
when run, you must cause it, when started running for such
|
||||
interactive use in the most ordinary way, to print or display an
|
||||
announcement including an appropriate copyright notice and a
|
||||
notice that there is no warranty (or else, saying that you provide
|
||||
a warranty) and that users may redistribute the program under
|
||||
these conditions, and telling the user how to view a copy of this
|
||||
License. (Exception: if the Program itself is interactive but
|
||||
does not normally print such an announcement, your work based on
|
||||
the Program is not required to print an announcement.)
|
||||
|
||||
These requirements apply to the modified work as a whole. If
|
||||
identifiable sections of that work are not derived from the Program,
|
||||
and can be reasonably considered independent and separate works in
|
||||
themselves, then this License, and its terms, do not apply to those
|
||||
sections when you distribute them as separate works. But when you
|
||||
distribute the same sections as part of a whole which is a work based
|
||||
on the Program, the distribution of the whole must be on the terms of
|
||||
this License, whose permissions for other licensees extend to the
|
||||
entire whole, and thus to each and every part regardless of who wrote it.
|
||||
|
||||
Thus, it is not the intent of this section to claim rights or contest
|
||||
your rights to work written entirely by you; rather, the intent is to
|
||||
exercise the right to control the distribution of derivative or
|
||||
collective works based on the Program.
|
||||
|
||||
In addition, mere aggregation of another work not based on the Program
|
||||
with the Program (or with a work based on the Program) on a volume of
|
||||
a storage or distribution medium does not bring the other work under
|
||||
the scope of this License.
|
||||
|
||||
3. You may copy and distribute the Program (or a work based on it,
|
||||
under Section 2) in object code or executable form under the terms of
|
||||
Sections 1 and 2 above provided that you also do one of the following:
|
||||
|
||||
a) Accompany it with the complete corresponding machine-readable
|
||||
source code, which must be distributed under the terms of Sections
|
||||
1 and 2 above on a medium customarily used for software interchange; or,
|
||||
|
||||
b) Accompany it with a written offer, valid for at least three
|
||||
years, to give any third party, for a charge no more than your
|
||||
cost of physically performing source distribution, a complete
|
||||
machine-readable copy of the corresponding source code, to be
|
||||
distributed under the terms of Sections 1 and 2 above on a medium
|
||||
customarily used for software interchange; or,
|
||||
|
||||
c) Accompany it with the information you received as to the offer
|
||||
to distribute corresponding source code. (This alternative is
|
||||
allowed only for noncommercial distribution and only if you
|
||||
received the program in object code or executable form with such
|
||||
an offer, in accord with Subsection b above.)
|
||||
|
||||
The source code for a work means the preferred form of the work for
|
||||
making modifications to it. For an executable work, complete source
|
||||
code means all the source code for all modules it contains, plus any
|
||||
associated interface definition files, plus the scripts used to
|
||||
control compilation and installation of the executable. However, as a
|
||||
special exception, the source code distributed need not include
|
||||
anything that is normally distributed (in either source or binary
|
||||
form) with the major components (compiler, kernel, and so on) of the
|
||||
operating system on which the executable runs, unless that component
|
||||
itself accompanies the executable.
|
||||
|
||||
If distribution of executable or object code is made by offering
|
||||
access to copy from a designated place, then offering equivalent
|
||||
access to copy the source code from the same place counts as
|
||||
distribution of the source code, even though third parties are not
|
||||
compelled to copy the source along with the object code.
|
||||
|
||||
4. You may not copy, modify, sublicense, or distribute the Program
|
||||
except as expressly provided under this License. Any attempt
|
||||
otherwise to copy, modify, sublicense or distribute the Program is
|
||||
void, and will automatically terminate your rights under this License.
|
||||
However, parties who have received copies, or rights, from you under
|
||||
this License will not have their licenses terminated so long as such
|
||||
parties remain in full compliance.
|
||||
|
||||
5. You are not required to accept this License, since you have not
|
||||
signed it. However, nothing else grants you permission to modify or
|
||||
distribute the Program or its derivative works. These actions are
|
||||
prohibited by law if you do not accept this License. Therefore, by
|
||||
modifying or distributing the Program (or any work based on the
|
||||
Program), you indicate your acceptance of this License to do so, and
|
||||
all its terms and conditions for copying, distributing or modifying
|
||||
the Program or works based on it.
|
||||
|
||||
6. Each time you redistribute the Program (or any work based on the
|
||||
Program), the recipient automatically receives a license from the
|
||||
original licensor to copy, distribute or modify the Program subject to
|
||||
these terms and conditions. You may not impose any further
|
||||
restrictions on the recipients' exercise of the rights granted herein.
|
||||
You are not responsible for enforcing compliance by third parties to
|
||||
this License.
|
||||
|
||||
7. If, as a consequence of a court judgment or allegation of patent
|
||||
infringement or for any other reason (not limited to patent issues),
|
||||
conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot
|
||||
distribute so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you
|
||||
may not distribute the Program at all. For example, if a patent
|
||||
license would not permit royalty-free redistribution of the Program by
|
||||
all those who receive copies directly or indirectly through you, then
|
||||
the only way you could satisfy both it and this License would be to
|
||||
refrain entirely from distribution of the Program.
|
||||
|
||||
If any portion of this section is held invalid or unenforceable under
|
||||
any particular circumstance, the balance of the section is intended to
|
||||
apply and the section as a whole is intended to apply in other
|
||||
circumstances.
|
||||
|
||||
It is not the purpose of this section to induce you to infringe any
|
||||
patents or other property right claims or to contest validity of any
|
||||
such claims; this section has the sole purpose of protecting the
|
||||
integrity of the free software distribution system, which is
|
||||
implemented by public license practices. Many people have made
|
||||
generous contributions to the wide range of software distributed
|
||||
through that system in reliance on consistent application of that
|
||||
system; it is up to the author/donor to decide if he or she is willing
|
||||
to distribute software through any other system and a licensee cannot
|
||||
impose that choice.
|
||||
|
||||
This section is intended to make thoroughly clear what is believed to
|
||||
be a consequence of the rest of this License.
|
||||
|
||||
8. If the distribution and/or use of the Program is restricted in
|
||||
certain countries either by patents or by copyrighted interfaces, the
|
||||
original copyright holder who places the Program under this License
|
||||
may add an explicit geographical distribution limitation excluding
|
||||
those countries, so that distribution is permitted only in or among
|
||||
countries not thus excluded. In such case, this License incorporates
|
||||
the limitation as if written in the body of this License.
|
||||
|
||||
9. The Free Software Foundation may publish revised and/or new versions
|
||||
of the General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Program
|
||||
specifies a version number of this License which applies to it and "any
|
||||
later version", you have the option of following the terms and conditions
|
||||
either of that version or of any later version published by the Free
|
||||
Software Foundation. If the Program does not specify a version number of
|
||||
this License, you may choose any version ever published by the Free Software
|
||||
Foundation.
|
||||
|
||||
10. If you wish to incorporate parts of the Program into other free
|
||||
programs whose distribution conditions are different, write to the author
|
||||
to ask for permission. For software which is copyrighted by the Free
|
||||
Software Foundation, write to the Free Software Foundation; we sometimes
|
||||
make exceptions for this. Our decision will be guided by the two goals
|
||||
of preserving the free status of all derivatives of our free software and
|
||||
of promoting the sharing and reuse of software generally.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
|
||||
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
|
||||
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
|
||||
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
|
||||
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
|
||||
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
|
||||
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
|
||||
REPAIR OR CORRECTION.
|
||||
|
||||
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
|
||||
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
|
||||
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
|
||||
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
|
||||
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
|
||||
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
convey the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program is interactive, make it output a short notice like this
|
||||
when it starts in an interactive mode:
|
||||
|
||||
Gnomovision version 69, Copyright (C) year name of author
|
||||
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, the commands you use may
|
||||
be called something other than `show w' and `show c'; they could even be
|
||||
mouse-clicks or menu items--whatever suits your program.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or your
|
||||
school, if any, to sign a "copyright disclaimer" for the program, if
|
||||
necessary. Here is a sample; alter the names:
|
||||
|
||||
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
|
||||
`Gnomovision' (which makes passes at compilers) written by James Hacker.
|
||||
|
||||
<signature of Ty Coon>, 1 April 1989
|
||||
Ty Coon, President of Vice
|
||||
|
||||
This General Public License does not permit incorporating your program into
|
||||
proprietary programs. If your program is a subroutine library, you may
|
||||
consider it more useful to permit linking proprietary applications with the
|
||||
library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License.
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
# go-lzo
|
||||
|
||||
[](https://travis-ci.org/rasky/go-lzo)
|
||||
[](https://coveralls.io/github/rasky/go-lzo?branch=master)
|
||||
|
||||
Native LZO1X implementation in Golang
|
||||
|
||||
This code has been written using the original LZO1X source code as a reference,
|
||||
to study and understand the algorithms. Both the LZO1X-1 and LZO1X-999
|
||||
algorithms are implemented. These are the most popular of the whole LZO suite
|
||||
of algorithms.
|
||||
|
||||
Being a straightforward port of the original source code, it shares the same
|
||||
license (GPLv2) as I can't possibly claim any copyright on it.
|
||||
|
||||
I plan to eventually reimplement LZO1X-1 from scratch. At that point, I will be
|
||||
also changing license.
|
||||
|
||||
# Benchmarks
|
||||
|
||||
These are the benchmarks obtained running the testsuite over the Canterbury
|
||||
corpus for the available compressor levels:
|
||||
|
||||
Compressor | Level | Original | Compressed | Factor | Time | Speed
|
||||
-----------|-------|----------|------------|--------|------|------
|
||||
LZO1X-1 | - | 18521760 | 8957481 | 51.6% | 0.16s | 109MiB/s
|
||||
LZO1X-999 | 1 | 18521760 | 8217347 | 55.6% | 1.38s | 13MiB/s
|
||||
LZO1X-999 | 2 | 18521760 | 7724879 | 58.3% | 1.50s | 12MiB/s
|
||||
LZO1X-999 | 3 | 18521760 | 7384377 | 60.1% | 1.68s | 10MiB/s
|
||||
LZO1X-999 | 4 | 18521760 | 7266674 | 60.8% | 1.69s | 10MiB/s
|
||||
LZO1X-999 | 5 | 18521760 | 6979879 | 62.3% | 2.75s | 6.4MiB/s
|
||||
LZO1X-999 | 6 | 18521760 | 6938593 | 62.5% | 4.53s | 3.9MiB/s
|
||||
LZO1X-999 | 7 | 18521760 | 6905362 | 62.7% | 6.94s | 2.5MiB/s
|
||||
LZO1X-999 | 8 | 18521760 | 6713477 | 63.8% | 20.96s | 863KiB/s
|
||||
LZO1X-999 | 9 | 18521760 | 6712069 | 63.8% | 22.82s | 792KiB/s
|
||||
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
package lzo
|
||||
|
||||
func appendMulti(out []byte, t int) []byte {
|
||||
for t > 255 {
|
||||
out = append(out, 0)
|
||||
t -= 255
|
||||
}
|
||||
out = append(out, byte(t))
|
||||
return out
|
||||
}
|
||||
|
||||
func compress(in []byte) (out []byte, sz int) {
|
||||
var m_off int
|
||||
in_len := len(in)
|
||||
ip_len := in_len - m2_MAX_LEN - 5
|
||||
dict := make([]int32, 1<<d_BITS)
|
||||
ii := 0
|
||||
ip := 4
|
||||
for {
|
||||
key := int(in[ip+3])
|
||||
key = (key << 6) ^ int(in[ip+2])
|
||||
key = (key << 5) ^ int(in[ip+1])
|
||||
key = (key << 5) ^ int(in[ip+0])
|
||||
dindex := ((0x21 * key) >> 5) & d_MASK
|
||||
m_pos := int(dict[dindex]) - 1
|
||||
if m_pos < 0 {
|
||||
goto literal
|
||||
}
|
||||
if ip == m_pos || (ip-m_pos) > m4_MAX_OFFSET {
|
||||
goto literal
|
||||
}
|
||||
m_off = ip - m_pos
|
||||
if m_off <= m2_MAX_OFFSET || in[m_pos+3] == in[ip+3] {
|
||||
goto try_match
|
||||
}
|
||||
|
||||
dindex = (dindex & (d_MASK & 0x7ff)) ^ (d_HIGH | 0x1f)
|
||||
m_pos = int(dict[dindex]) - 1
|
||||
if m_pos < 0 {
|
||||
goto literal
|
||||
}
|
||||
if ip == m_pos || (ip-m_pos) > m4_MAX_OFFSET {
|
||||
goto literal
|
||||
}
|
||||
m_off = ip - m_pos
|
||||
if m_off <= m2_MAX_OFFSET || in[m_pos+3] == in[ip+3] {
|
||||
goto try_match
|
||||
}
|
||||
|
||||
goto literal
|
||||
|
||||
try_match:
|
||||
if in[m_pos] == in[ip] && in[m_pos+1] == in[ip+1] && in[m_pos+2] == in[ip+2] {
|
||||
goto match
|
||||
}
|
||||
|
||||
literal:
|
||||
dict[dindex] = int32(ip + 1)
|
||||
ip += 1 + (ip-ii)>>5
|
||||
if ip >= ip_len {
|
||||
break
|
||||
}
|
||||
continue
|
||||
|
||||
match:
|
||||
dict[dindex] = int32(ip + 1)
|
||||
if ip != ii {
|
||||
t := ip - ii
|
||||
if t <= 3 {
|
||||
out[len(out)-2] |= byte(t)
|
||||
} else if t <= 18 {
|
||||
out = append(out, byte(t-3))
|
||||
} else {
|
||||
out = append(out, 0)
|
||||
out = appendMulti(out, t-18)
|
||||
}
|
||||
|
||||
out = append(out, in[ii:ii+t]...)
|
||||
ii += t
|
||||
}
|
||||
|
||||
var i int
|
||||
ip += 3
|
||||
for i = 3; i < 9; i++ {
|
||||
ip++
|
||||
if in[m_pos+i] != in[ip-1] {
|
||||
break
|
||||
}
|
||||
}
|
||||
if i < 9 {
|
||||
ip--
|
||||
m_len := ip - ii
|
||||
if m_off <= m2_MAX_OFFSET {
|
||||
m_off -= 1
|
||||
out = append(out,
|
||||
byte((((m_len - 1) << 5) | ((m_off & 7) << 2))),
|
||||
byte((m_off >> 3)))
|
||||
} else if m_off <= m3_MAX_OFFSET {
|
||||
m_off -= 1
|
||||
out = append(out,
|
||||
byte(m3_MARKER|(m_len-2)),
|
||||
byte((m_off&63)<<2),
|
||||
byte(m_off>>6))
|
||||
} else {
|
||||
m_off -= 0x4000
|
||||
out = append(out,
|
||||
byte(m4_MARKER|((m_off&0x4000)>>11)|(m_len-2)),
|
||||
byte((m_off&63)<<2),
|
||||
byte(m_off>>6))
|
||||
}
|
||||
} else {
|
||||
m := m_pos + m2_MAX_LEN + 1
|
||||
for ip < in_len && in[m] == in[ip] {
|
||||
m++
|
||||
ip++
|
||||
}
|
||||
m_len := ip - ii
|
||||
if m_off <= m3_MAX_OFFSET {
|
||||
m_off -= 1
|
||||
if m_len <= 33 {
|
||||
out = append(out, byte(m3_MARKER|(m_len-2)))
|
||||
} else {
|
||||
m_len -= 33
|
||||
out = append(out, byte(m3_MARKER|0))
|
||||
out = appendMulti(out, m_len)
|
||||
}
|
||||
} else {
|
||||
m_off -= 0x4000
|
||||
if m_len <= m4_MAX_LEN {
|
||||
out = append(out, byte(m4_MARKER|((m_off&0x4000)>>11)|(m_len-2)))
|
||||
} else {
|
||||
m_len -= m4_MAX_LEN
|
||||
out = append(out, byte(m4_MARKER|((m_off&0x4000)>>11)))
|
||||
out = appendMulti(out, m_len)
|
||||
}
|
||||
}
|
||||
out = append(out, byte((m_off&63)<<2), byte(m_off>>6))
|
||||
}
|
||||
|
||||
ii = ip
|
||||
if ip >= ip_len {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
sz = in_len - ii
|
||||
return
|
||||
}
|
||||
|
||||
// Compress an input buffer with LZO1X
|
||||
func Compress1X(in []byte) (out []byte) {
|
||||
var t int
|
||||
|
||||
in_len := len(in)
|
||||
if in_len <= m2_MAX_LEN+5 {
|
||||
t = in_len
|
||||
} else {
|
||||
out, t = compress(in)
|
||||
}
|
||||
|
||||
if t > 0 {
|
||||
ii := in_len - t
|
||||
if len(out) == 0 && t <= 238 {
|
||||
out = append(out, byte(17+t))
|
||||
} else if t <= 3 {
|
||||
out[len(out)-2] |= byte(t)
|
||||
} else if t <= 18 {
|
||||
out = append(out, byte(t-3))
|
||||
} else {
|
||||
out = append(out, 0)
|
||||
out = appendMulti(out, t-18)
|
||||
}
|
||||
out = append(out, in[ii:ii+t]...)
|
||||
}
|
||||
|
||||
out = append(out, m4_MARKER|1, 0, 0)
|
||||
return
|
||||
}
|
||||
+416
@@ -0,0 +1,416 @@
|
||||
package lzo
|
||||
|
||||
type compressor struct {
|
||||
in []byte
|
||||
ip int
|
||||
bp int
|
||||
|
||||
// stats
|
||||
matchBytes int
|
||||
litBytes int
|
||||
lazy int
|
||||
|
||||
r1lit int
|
||||
r2lit int
|
||||
m1am uint
|
||||
m2m uint
|
||||
m1bm uint
|
||||
m3m uint
|
||||
m4m uint
|
||||
lit1r uint
|
||||
lit2r uint
|
||||
lit3r uint
|
||||
|
||||
r1mlen int
|
||||
|
||||
lastmlen int
|
||||
lastmoff int
|
||||
textsize uint
|
||||
mlen int
|
||||
moff int
|
||||
look uint
|
||||
}
|
||||
|
||||
func (ctx *compressor) codeMatch(out []byte, mlen int, moff int) []byte {
|
||||
xlen := mlen
|
||||
xoff := moff
|
||||
ctx.matchBytes += mlen
|
||||
|
||||
switch {
|
||||
case mlen == 2:
|
||||
if moff > m1_MAX_OFFSET {
|
||||
panic("codeMatch: mlen 2: moff error")
|
||||
}
|
||||
if ctx.r1lit < 1 || ctx.r1lit >= 4 {
|
||||
panic("codeMatch: mlen 2: r1lit error")
|
||||
}
|
||||
moff -= 1
|
||||
out = append(out,
|
||||
m1_MARKER|byte((moff&3)<<2),
|
||||
byte(moff>>2))
|
||||
ctx.m1am++
|
||||
case mlen <= m2_MAX_LEN && moff <= m2_MAX_OFFSET:
|
||||
if mlen < 3 {
|
||||
panic("codeMatch: m2: mlen error")
|
||||
}
|
||||
moff -= 1
|
||||
out = append(out,
|
||||
byte((mlen-1)<<5|(moff&7)<<2),
|
||||
byte(moff>>3))
|
||||
if out[len(out)-2] < m2_MARKER {
|
||||
panic("codeMatch: m2: invalid marker")
|
||||
}
|
||||
ctx.m2m++
|
||||
case mlen == m2_MIN_LEN && moff <= mX_MAX_OFFSET && ctx.r1lit >= 4:
|
||||
if mlen != 3 {
|
||||
panic("codeMatch: m2min: invalid mlen")
|
||||
}
|
||||
if moff <= m2_MAX_OFFSET {
|
||||
panic("codeMatch: m2min: invalid moff")
|
||||
}
|
||||
moff -= 1 + m2_MAX_OFFSET
|
||||
out = append(out,
|
||||
byte(m1_MARKER|((moff&3)<<2)),
|
||||
byte(moff>>2))
|
||||
ctx.m1bm++
|
||||
case moff <= m3_MAX_OFFSET:
|
||||
if mlen < 3 {
|
||||
panic("codeMatch: m3max: invalid mlen")
|
||||
}
|
||||
moff -= 1
|
||||
if mlen <= m3_MAX_LEN {
|
||||
out = append(out, byte(m3_MARKER|(mlen-2)))
|
||||
} else {
|
||||
mlen -= m3_MAX_LEN
|
||||
out = append(out, byte(m3_MARKER|0))
|
||||
out = appendMulti(out, mlen)
|
||||
}
|
||||
out = append(out, byte(moff<<2), byte(moff>>6))
|
||||
ctx.m3m++
|
||||
default:
|
||||
if mlen < 3 {
|
||||
panic("codeMatch: default: invalid mlen")
|
||||
}
|
||||
if moff <= 0x4000 || moff >= 0xc000 {
|
||||
panic("codeMatch: default: invalid moff")
|
||||
}
|
||||
moff -= 0x4000
|
||||
k := (moff & 0x4000) >> 11
|
||||
if mlen <= m4_MAX_LEN {
|
||||
out = append(out, byte(m4_MARKER|k|(mlen-2)))
|
||||
} else {
|
||||
mlen -= m4_MAX_LEN
|
||||
out = append(out, byte(m4_MARKER|k|0))
|
||||
out = appendMulti(out, mlen)
|
||||
}
|
||||
out = append(out, byte(moff<<2), byte(moff>>6))
|
||||
ctx.m4m++
|
||||
}
|
||||
|
||||
ctx.lastmlen = xlen
|
||||
ctx.lastmoff = xoff
|
||||
return out
|
||||
}
|
||||
|
||||
func (ctx *compressor) storeRun(out []byte, ii int, t int) []byte {
|
||||
ctx.litBytes += t
|
||||
|
||||
if len(out) == 0 && t <= 238 {
|
||||
out = append(out, byte(17+t))
|
||||
} else if t <= 3 {
|
||||
out[len(out)-2] |= byte(t)
|
||||
ctx.lit1r++
|
||||
} else if t <= 18 {
|
||||
out = append(out, byte(t-3))
|
||||
ctx.lit2r++
|
||||
} else {
|
||||
out = append(out, 0)
|
||||
out = appendMulti(out, t-18)
|
||||
ctx.lit3r++
|
||||
}
|
||||
|
||||
out = append(out, ctx.in[ii:ii+t]...)
|
||||
return out
|
||||
}
|
||||
|
||||
func (ctx *compressor) codeRun(out []byte, ii int, lit int, mlen int) []byte {
|
||||
if lit > 0 {
|
||||
if mlen < 2 {
|
||||
panic("codeRun: invalid mlen")
|
||||
}
|
||||
out = ctx.storeRun(out, ii, lit)
|
||||
ctx.r1mlen = mlen
|
||||
ctx.r1lit = lit
|
||||
} else {
|
||||
if mlen < 3 {
|
||||
panic("codeRun: invalid mlen")
|
||||
}
|
||||
ctx.r1mlen = 0
|
||||
ctx.r1lit = 0
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (ctx *compressor) lenOfCodedMatch(mlen int, moff int, lit int) int {
|
||||
switch {
|
||||
case mlen < 2:
|
||||
return 0
|
||||
case mlen == 2:
|
||||
if moff <= m1_MAX_OFFSET && lit > 0 && lit < 4 {
|
||||
return 2
|
||||
}
|
||||
return 0
|
||||
case mlen <= m2_MAX_LEN && moff <= m2_MAX_OFFSET:
|
||||
return 2
|
||||
case mlen == m2_MIN_LEN && moff <= mX_MAX_OFFSET && lit >= 4:
|
||||
return 2
|
||||
case moff <= m3_MAX_OFFSET:
|
||||
if mlen <= m3_MAX_LEN {
|
||||
return 3
|
||||
}
|
||||
n := 4
|
||||
mlen -= m3_MAX_LEN
|
||||
for mlen > 255 {
|
||||
mlen -= 255
|
||||
n++
|
||||
}
|
||||
return n
|
||||
case moff <= m4_MAX_OFFSET:
|
||||
if mlen <= m4_MAX_LEN {
|
||||
return 3
|
||||
}
|
||||
n := 4
|
||||
mlen -= m4_MAX_LEN
|
||||
for mlen > 255 {
|
||||
mlen -= 255
|
||||
n++
|
||||
}
|
||||
return n
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *compressor) minGain(ahead int,
|
||||
lit1, lit2 int, l1, l2, l3 int) int {
|
||||
|
||||
if ahead <= 0 {
|
||||
panic("minGain: invalid ahead")
|
||||
}
|
||||
mingain := int(ahead)
|
||||
if lit1 <= 3 {
|
||||
if lit2 > 3 {
|
||||
mingain += 2
|
||||
}
|
||||
} else if lit1 <= 18 {
|
||||
if lit2 > 18 {
|
||||
mingain += 1
|
||||
}
|
||||
}
|
||||
|
||||
mingain += int((l2 - l1) * 2)
|
||||
if l3 != 0 {
|
||||
mingain -= int((ahead - l3) * 2)
|
||||
}
|
||||
if mingain < 0 {
|
||||
mingain = 0
|
||||
}
|
||||
return mingain
|
||||
}
|
||||
|
||||
type parms struct {
|
||||
TryLazy int
|
||||
GoodLen uint
|
||||
MaxLazy uint
|
||||
NiceLen uint
|
||||
MaxChain uint
|
||||
Flags uint32
|
||||
}
|
||||
|
||||
func compress999(in []byte, p parms) []byte {
|
||||
ctx := compressor{}
|
||||
swd := swd{}
|
||||
|
||||
if p.TryLazy < 0 {
|
||||
p.TryLazy = 1
|
||||
}
|
||||
if p.GoodLen == 0 {
|
||||
p.GoodLen = 32
|
||||
}
|
||||
if p.MaxLazy == 0 {
|
||||
p.MaxLazy = 32
|
||||
}
|
||||
if p.MaxChain == 0 {
|
||||
p.MaxChain = cSWD_MAX_CHAIN
|
||||
}
|
||||
|
||||
ctx.in = in
|
||||
|
||||
out := make([]byte, 0, len(in)/2)
|
||||
ii := 0
|
||||
lit := 0
|
||||
|
||||
ctx.initMatch(&swd, p.Flags)
|
||||
if p.MaxChain > 0 {
|
||||
swd.MaxChain = p.MaxChain
|
||||
}
|
||||
if p.NiceLen > 0 {
|
||||
swd.NiceLength = p.NiceLen
|
||||
}
|
||||
|
||||
ctx.findMatch(&swd, 0, 0)
|
||||
for ctx.look > 0 {
|
||||
mlen := ctx.mlen
|
||||
moff := ctx.moff
|
||||
if ctx.bp != ctx.ip-int(ctx.look) {
|
||||
panic("assert: compress: invalid bp")
|
||||
}
|
||||
if ctx.bp < 0 {
|
||||
panic("assert: compress: negative bp")
|
||||
}
|
||||
if lit == 0 {
|
||||
ii = ctx.bp
|
||||
}
|
||||
if ii+lit != ctx.bp {
|
||||
panic("assert: compress: invalid ii")
|
||||
}
|
||||
if swd.BChar != int(ctx.in[ctx.bp]) {
|
||||
panic("assert: compress: invalid bchar")
|
||||
}
|
||||
|
||||
if mlen < 2 ||
|
||||
(mlen == 2 && (moff > m1_MAX_OFFSET || lit == 0 || lit >= 4)) ||
|
||||
(mlen == 2 && len(out) == 0) ||
|
||||
(len(out) == 0 && lit == 0) {
|
||||
// literal
|
||||
mlen = 0
|
||||
} else if mlen == m2_MIN_LEN {
|
||||
if moff > mX_MAX_OFFSET && lit >= 4 {
|
||||
mlen = 0
|
||||
}
|
||||
}
|
||||
|
||||
if mlen == 0 {
|
||||
// literal
|
||||
lit++
|
||||
swd.MaxChain = p.MaxChain
|
||||
ctx.findMatch(&swd, 1, 0)
|
||||
continue
|
||||
}
|
||||
|
||||
// a match
|
||||
if swd.UseBestOff {
|
||||
mlen, moff = ctx.betterMatch(&swd, mlen, moff)
|
||||
}
|
||||
|
||||
ctx.assertMatch(&swd, mlen, moff)
|
||||
|
||||
// check if we want to try a lazy match
|
||||
ahead := 0
|
||||
l1 := 0
|
||||
maxahead := 0
|
||||
if p.TryLazy != 0 && mlen < int(p.MaxLazy) {
|
||||
l1 = ctx.lenOfCodedMatch(mlen, moff, lit)
|
||||
if l1 == 0 {
|
||||
panic("assert: compress: invalid len of coded match")
|
||||
}
|
||||
maxahead = p.TryLazy
|
||||
if maxahead > l1-1 {
|
||||
maxahead = l1 - 1
|
||||
}
|
||||
}
|
||||
|
||||
matchdone := false
|
||||
for ahead < maxahead && int(ctx.look) > mlen {
|
||||
if mlen >= int(p.GoodLen) {
|
||||
swd.MaxChain = p.MaxChain >> 2
|
||||
} else {
|
||||
swd.MaxChain = p.MaxChain
|
||||
}
|
||||
ctx.findMatch(&swd, 1, 0)
|
||||
ahead++
|
||||
if ctx.look <= 0 {
|
||||
panic("assert: compress: invalid look")
|
||||
}
|
||||
if ii+lit+ahead != ctx.bp {
|
||||
panic("assert: compress: invalid bp")
|
||||
}
|
||||
if ctx.mlen < mlen {
|
||||
continue
|
||||
}
|
||||
if ctx.mlen == mlen && ctx.moff >= moff {
|
||||
continue
|
||||
}
|
||||
if swd.UseBestOff {
|
||||
ctx.mlen, ctx.moff = ctx.betterMatch(&swd, ctx.mlen, ctx.moff)
|
||||
}
|
||||
l2 := ctx.lenOfCodedMatch(ctx.mlen, ctx.moff, lit+ahead)
|
||||
if l2 == 0 {
|
||||
continue
|
||||
}
|
||||
l3 := 0
|
||||
if len(out) > 0 {
|
||||
l3 = ctx.lenOfCodedMatch(ahead, moff, lit)
|
||||
}
|
||||
mingain := ctx.minGain(ahead, lit, lit+ahead, l1, l2, l3)
|
||||
if ctx.mlen >= mlen+mingain {
|
||||
ctx.lazy++
|
||||
ctx.assertMatch(&swd, ctx.mlen, ctx.moff)
|
||||
|
||||
if l3 > 0 {
|
||||
out = ctx.codeRun(out, ii, lit, ahead)
|
||||
lit = 0
|
||||
out = ctx.codeMatch(out, ahead, moff)
|
||||
} else {
|
||||
lit += ahead
|
||||
if ii+lit != ctx.bp {
|
||||
panic("assert: compress: invalid bp after l3")
|
||||
}
|
||||
}
|
||||
matchdone = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !matchdone {
|
||||
if ii+lit+ahead != ctx.bp {
|
||||
panic("assert: compress: invalid bp out of for loop")
|
||||
}
|
||||
|
||||
out = ctx.codeRun(out, ii, lit, mlen)
|
||||
lit = 0
|
||||
out = ctx.codeMatch(out, mlen, moff)
|
||||
swd.MaxChain = p.MaxChain
|
||||
ctx.findMatch(&swd, uint(mlen), uint(1+ahead))
|
||||
}
|
||||
}
|
||||
|
||||
if lit > 0 {
|
||||
out = ctx.storeRun(out, ii, lit)
|
||||
}
|
||||
out = append(out, m4_MARKER|1, 0, 0)
|
||||
if ctx.litBytes+ctx.matchBytes != len(ctx.in) {
|
||||
panic("assert: compress999: not processed full input")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var fixedLevels = [...]parms{
|
||||
{0, 0, 0, 8, 4, 0},
|
||||
{0, 0, 0, 16, 8, 0},
|
||||
{0, 0, 0, 32, 16, 0},
|
||||
{1, 4, 4, 16, 16, 0},
|
||||
{1, 8, 16, 32, 32, 0},
|
||||
{1, 8, 16, 128, 128, 0},
|
||||
{2, 8, 32, 128, 256, 0},
|
||||
{2, 32, 128, cSWD_F, 2048, 1},
|
||||
{2, cSWD_F, cSWD_F, cSWD_F, 4096, 1},
|
||||
}
|
||||
|
||||
func Compress1X999Level(in []byte, level int) []byte {
|
||||
return compress999(in, fixedLevels[level-1])
|
||||
}
|
||||
|
||||
func Compress1X999(in []byte) []byte {
|
||||
return Compress1X999Level(in, 9)
|
||||
}
|
||||
+289
@@ -0,0 +1,289 @@
|
||||
package lzo
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
var (
|
||||
InputUnderrun = errors.New("input underrun")
|
||||
LookBehindUnderrun = errors.New("lookbehind underrun")
|
||||
)
|
||||
|
||||
type reader struct {
|
||||
r io.Reader
|
||||
len int
|
||||
buf [4096]byte
|
||||
cur []byte
|
||||
Err error
|
||||
}
|
||||
|
||||
func newReader(r io.Reader, inlen int) *reader {
|
||||
if inlen == 0 {
|
||||
inlen = -1
|
||||
}
|
||||
in := &reader{r: r, len: inlen}
|
||||
in.Rebuffer()
|
||||
return in
|
||||
}
|
||||
|
||||
// Read more data from the underlying reader and put it into the buffer.
|
||||
// Also makes sure there is always at least 32 bytes in the buffer, so that
|
||||
// in the main loop we can avoid checking for the end of buffer.
|
||||
func (in *reader) Rebuffer() {
|
||||
const RBUF_WND = 32
|
||||
var rbuf [RBUF_WND]byte
|
||||
|
||||
if len(in.cur) > RBUF_WND || in.len == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
rb := rbuf[:len(in.cur)]
|
||||
copy(rb, in.cur)
|
||||
in.cur = in.buf[:]
|
||||
copy(in.cur, rb)
|
||||
|
||||
cur := in.cur[len(rb):]
|
||||
if in.len >= 0 && len(cur) > in.len {
|
||||
cur = cur[:in.len]
|
||||
}
|
||||
n, err := in.r.Read(cur)
|
||||
if err != nil {
|
||||
// If EOF is returned, treat it as error only if there are no further
|
||||
// bytes in the window. Otherwise, let's postpone because those bytes
|
||||
// could contain the terminator.
|
||||
if err != io.EOF || len(rb) == 0 {
|
||||
in.Err = err
|
||||
in.cur = nil
|
||||
}
|
||||
}
|
||||
in.cur = in.cur[:len(rb)+n]
|
||||
if in.len >= 0 {
|
||||
in.len -= n
|
||||
}
|
||||
}
|
||||
|
||||
func (in *reader) ReadAppend(out *[]byte, n int) {
|
||||
for n > 0 {
|
||||
m := len(in.cur)
|
||||
if m > n {
|
||||
m = n
|
||||
}
|
||||
*out = append(*out, in.cur[:m]...)
|
||||
in.cur = in.cur[m:]
|
||||
n -= m
|
||||
if len(in.cur) == 0 {
|
||||
in.Rebuffer()
|
||||
if len(in.cur) == 0 {
|
||||
in.Err = io.EOF
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (in *reader) ReadU8() (ch byte) {
|
||||
ch = in.cur[0]
|
||||
in.cur = in.cur[1:]
|
||||
return
|
||||
}
|
||||
|
||||
func (in *reader) ReadU16() int {
|
||||
b0 := in.cur[0]
|
||||
b1 := in.cur[1]
|
||||
in.cur = in.cur[2:]
|
||||
return int(b0) + int(b1)<<8
|
||||
}
|
||||
|
||||
func (in *reader) ReadMulti(base int) (b int) {
|
||||
for {
|
||||
for i := 0; i < len(in.cur); i++ {
|
||||
v := in.cur[i]
|
||||
if v == 0 {
|
||||
b += 255
|
||||
} else {
|
||||
b += int(v) + base
|
||||
in.cur = in.cur[i+1:]
|
||||
return
|
||||
}
|
||||
}
|
||||
in.cur = in.cur[0:0]
|
||||
in.Rebuffer()
|
||||
if len(in.cur) == 0 {
|
||||
in.Err = io.EOF
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func copyMatch(out *[]byte, m_pos int, n int) {
|
||||
if m_pos+n > len(*out) {
|
||||
// fmt.Println("copy match WITH OVERLAP!")
|
||||
for i := 0; i < n; i++ {
|
||||
*out = append(*out, (*out)[m_pos])
|
||||
m_pos++
|
||||
}
|
||||
} else {
|
||||
// fmt.Println("copy match:", len(*out), m_pos, m_pos+n)
|
||||
*out = append(*out, (*out)[m_pos:m_pos+n]...)
|
||||
}
|
||||
}
|
||||
|
||||
// Decompress an input compressed with LZO1X.
|
||||
//
|
||||
// LZO1X has a stream terminator marker, so the decompression will always stop
|
||||
// when this marker is found.
|
||||
//
|
||||
// If inLen is not zero, it is expected to match the length of the compressed
|
||||
// input stream, and it is used to limit reads from the underlying reader; if
|
||||
// inLen is smaller than the real stream, the decompression will abort with an
|
||||
// error; if inLen is larger than the real stream, or if it is zero, the
|
||||
// decompression will succeed but more bytes than necessary might be read
|
||||
// from the underlying reader. If the reader returns EOF before the termination
|
||||
// marker is found, the decompression aborts and EOF is returned.
|
||||
//
|
||||
// outLen is optional; if it's not zero, it is used as a hint to preallocate the
|
||||
// output buffer to increase performance of the decompression.
|
||||
func Decompress1X(r io.Reader, inLen int, outLen int) (out []byte, err error) {
|
||||
var t, m_pos int
|
||||
var last2 byte
|
||||
|
||||
defer func() {
|
||||
// To gain performance, we don't do any bounds checking while reading
|
||||
// the input, so if the decompressor reads past the end of the input
|
||||
// stream, a runtime error is raised. This saves about 7% of performance
|
||||
// as the reading functions are very hot in the decompressor.
|
||||
if r := recover(); r != nil {
|
||||
if re, ok := r.(runtime.Error); ok {
|
||||
if re.Error() == "runtime error: index out of range" {
|
||||
err = io.EOF
|
||||
return
|
||||
}
|
||||
}
|
||||
panic(r)
|
||||
}
|
||||
}()
|
||||
|
||||
out = make([]byte, 0, outLen)
|
||||
|
||||
in := newReader(r, inLen)
|
||||
ip := in.ReadU8()
|
||||
if ip > 17 {
|
||||
t = int(ip) - 17
|
||||
if t < 4 {
|
||||
goto match_next
|
||||
}
|
||||
in.ReadAppend(&out, t)
|
||||
// fmt.Println("begin:", string(out))
|
||||
goto first_literal_run
|
||||
}
|
||||
|
||||
begin_loop:
|
||||
t = int(ip)
|
||||
if t >= 16 {
|
||||
goto match
|
||||
}
|
||||
if t == 0 {
|
||||
t = in.ReadMulti(15)
|
||||
}
|
||||
in.ReadAppend(&out, t+3)
|
||||
// fmt.Println("readappend", t+3, string(out[len(out)-t-3:]))
|
||||
first_literal_run:
|
||||
ip = in.ReadU8()
|
||||
last2 = ip
|
||||
t = int(ip)
|
||||
if t >= 16 {
|
||||
goto match
|
||||
}
|
||||
m_pos = len(out) - (1 + m2_MAX_OFFSET)
|
||||
m_pos -= t >> 2
|
||||
ip = in.ReadU8()
|
||||
m_pos -= int(ip) << 2
|
||||
// fmt.Println("m_pos flr", m_pos, len(out), "\n", string(out))
|
||||
if m_pos < 0 {
|
||||
err = LookBehindUnderrun
|
||||
return
|
||||
}
|
||||
copyMatch(&out, m_pos, 3)
|
||||
goto match_done
|
||||
|
||||
match:
|
||||
in.Rebuffer()
|
||||
if in.Err != nil {
|
||||
err = in.Err
|
||||
return
|
||||
}
|
||||
t = int(ip)
|
||||
last2 = ip
|
||||
if t >= 64 {
|
||||
m_pos = len(out) - 1
|
||||
m_pos -= (t >> 2) & 7
|
||||
ip = in.ReadU8()
|
||||
m_pos -= int(ip) << 3
|
||||
// fmt.Println("m_pos t64", m_pos, t, int(ip))
|
||||
t = (t >> 5) - 1
|
||||
goto copy_match
|
||||
} else if t >= 32 {
|
||||
t &= 31
|
||||
if t == 0 {
|
||||
t = in.ReadMulti(31)
|
||||
}
|
||||
m_pos = len(out) - 1
|
||||
v16 := in.ReadU16()
|
||||
m_pos -= v16 >> 2
|
||||
last2 = byte(v16 & 0xFF)
|
||||
// fmt.Println("m_pos t32", m_pos)
|
||||
} else if t >= 16 {
|
||||
m_pos = len(out)
|
||||
m_pos -= (t & 8) << 11
|
||||
t &= 7
|
||||
if t == 0 {
|
||||
t = in.ReadMulti(7)
|
||||
}
|
||||
v16 := in.ReadU16()
|
||||
m_pos -= v16 >> 2
|
||||
if m_pos == len(out) {
|
||||
// fmt.Println("END", t, v16, m_pos)
|
||||
return
|
||||
}
|
||||
m_pos -= 0x4000
|
||||
last2 = byte(v16 & 0xFF)
|
||||
// fmt.Println("m_pos t16", m_pos)
|
||||
} else {
|
||||
m_pos = len(out) - 1
|
||||
m_pos -= t >> 2
|
||||
ip = in.ReadU8()
|
||||
m_pos -= int(ip) << 2
|
||||
if m_pos < 0 {
|
||||
err = LookBehindUnderrun
|
||||
return
|
||||
}
|
||||
// fmt.Println("m_pos tX", m_pos)
|
||||
copyMatch(&out, m_pos, 2)
|
||||
goto match_done
|
||||
}
|
||||
|
||||
copy_match:
|
||||
if m_pos < 0 {
|
||||
err = LookBehindUnderrun
|
||||
return
|
||||
}
|
||||
copyMatch(&out, m_pos, t+2)
|
||||
|
||||
match_done:
|
||||
t = int(last2 & 3)
|
||||
if t == 0 {
|
||||
goto match_end
|
||||
}
|
||||
match_next:
|
||||
// fmt.Println("read append finale:", t)
|
||||
in.ReadAppend(&out, t)
|
||||
ip = in.ReadU8()
|
||||
goto match
|
||||
|
||||
match_end:
|
||||
ip = in.ReadU8()
|
||||
goto begin_loop
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package lzo
|
||||
|
||||
const (
|
||||
m1_MAX_OFFSET = 0x0400
|
||||
m2_MAX_OFFSET = 0x0800
|
||||
m3_MAX_OFFSET = 0x4000
|
||||
m4_MAX_OFFSET = 0xbfff
|
||||
mX_MAX_OFFSET = m1_MAX_OFFSET + m2_MAX_OFFSET
|
||||
|
||||
m1_MIN_LEN = 2
|
||||
m1_MAX_LEN = 2
|
||||
m2_MIN_LEN = 3
|
||||
m2_MAX_LEN = 8
|
||||
m3_MIN_LEN = 3
|
||||
m3_MAX_LEN = 33
|
||||
m4_MIN_LEN = 3
|
||||
m4_MAX_LEN = 9
|
||||
|
||||
m1_MARKER = 0
|
||||
m2_MARKER = 64
|
||||
m3_MARKER = 32
|
||||
m4_MARKER = 16
|
||||
)
|
||||
|
||||
const (
|
||||
d_BITS = 14
|
||||
d_MASK = (1 << d_BITS) - 1
|
||||
d_HIGH = (d_MASK >> 1) + 1
|
||||
)
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// +build gofuzz
|
||||
|
||||
package lzo
|
||||
|
||||
import "bytes"
|
||||
|
||||
func Fuzz(data []byte) int {
|
||||
Decompress1X(bytes.NewBuffer(data), 0, 0)
|
||||
return 0
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
package lzo
|
||||
|
||||
func (ctx *compressor) initMatch(s *swd, flags uint32) {
|
||||
s.ctx = ctx
|
||||
s.init()
|
||||
if flags&1 != 0 {
|
||||
s.UseBestOff = true
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *compressor) findMatch(s *swd, thislen uint, skip uint) {
|
||||
if skip > 0 {
|
||||
if thislen < skip {
|
||||
panic("assert: findMatch: invalid thislen")
|
||||
}
|
||||
s.accept(thislen - skip)
|
||||
ctx.textsize += thislen - skip + 1
|
||||
} else {
|
||||
if thislen > 1 {
|
||||
panic("assert: findMatch: invalid thislen")
|
||||
}
|
||||
ctx.textsize += thislen - skip
|
||||
}
|
||||
|
||||
s.MLen = cSWD_THRESHOLD
|
||||
s.MOff = 0
|
||||
for i := 0; i < len(s.bestPos); i++ {
|
||||
s.bestPos[i] = 0
|
||||
}
|
||||
|
||||
s.findbest()
|
||||
ctx.mlen = int(s.MLen)
|
||||
ctx.moff = int(s.MOff)
|
||||
|
||||
s.getbyte()
|
||||
if s.BChar < 0 {
|
||||
ctx.look = 0
|
||||
ctx.mlen = 0
|
||||
} else {
|
||||
ctx.look = s.Look + 1
|
||||
}
|
||||
|
||||
ctx.bp = ctx.ip - int(ctx.look)
|
||||
}
|
||||
|
||||
func (ctx *compressor) betterMatch(s *swd, imlen, imoff int) (mlen int, moff int) {
|
||||
mlen, moff = imlen, imoff
|
||||
if mlen <= m2_MIN_LEN {
|
||||
return
|
||||
}
|
||||
if moff <= m2_MAX_OFFSET {
|
||||
return
|
||||
}
|
||||
|
||||
if moff > m2_MAX_OFFSET && mlen >= m2_MIN_LEN+1 && mlen <= m2_MAX_LEN+1 &&
|
||||
s.BestOff[mlen-1] > 0 && s.BestOff[mlen-1] <= m2_MAX_OFFSET {
|
||||
mlen -= 1
|
||||
moff = int(s.BestOff[mlen])
|
||||
return
|
||||
}
|
||||
|
||||
if moff > m3_MAX_OFFSET && mlen >= m4_MAX_LEN+1 && mlen <= m2_MAX_LEN+2 &&
|
||||
s.BestOff[mlen-2] > 0 && s.BestOff[mlen-2] <= m2_MAX_OFFSET {
|
||||
mlen -= 2
|
||||
moff = int(s.BestOff[mlen])
|
||||
return
|
||||
}
|
||||
|
||||
if moff > m3_MAX_OFFSET && mlen >= m4_MAX_LEN+1 && mlen <= m3_MAX_LEN+1 &&
|
||||
s.BestOff[mlen-1] > 0 && s.BestOff[mlen-1] <= m3_MAX_OFFSET {
|
||||
mlen -= 1
|
||||
moff = int(s.BestOff[mlen])
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func assertMemcmp(b1, b2 []byte, l int) {
|
||||
b1 = b1[:l]
|
||||
b2 = b2[:l]
|
||||
for i := 0; i < len(b1); i++ {
|
||||
if b1[i] != b2[i] {
|
||||
panic("assertMemcmp: dosn't match")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *compressor) assertMatch(s *swd, mlen, moff int) {
|
||||
if mlen < 2 {
|
||||
panic("assertMatch: invalid mlen")
|
||||
}
|
||||
if moff <= ctx.bp {
|
||||
if ctx.bp-moff+mlen >= ctx.ip {
|
||||
panic("assertMatch: invalid bp")
|
||||
}
|
||||
assertMemcmp(ctx.in[ctx.bp:], ctx.in[ctx.bp-moff:], mlen)
|
||||
} else {
|
||||
panic("dict should not exit")
|
||||
}
|
||||
}
|
||||
+349
@@ -0,0 +1,349 @@
|
||||
package lzo
|
||||
|
||||
const (
|
||||
cSWD_N = m4_MAX_OFFSET // ring buffer size
|
||||
cSWD_THRESHOLD = 1 // lower limit for match length
|
||||
cSWD_F = 2048 // upper limit for match length
|
||||
cSWD_BEST_OFF = m3_MAX_LEN + 1 // max(m2,m3,m4)+1
|
||||
cSWD_HSIZE = 16384
|
||||
cSWD_MAX_CHAIN = 2048
|
||||
)
|
||||
|
||||
type swd struct {
|
||||
// Public builtin
|
||||
SwdN uint
|
||||
SwdF uint
|
||||
SwdThreshold uint
|
||||
|
||||
// Public configuration
|
||||
MaxChain uint
|
||||
NiceLength uint
|
||||
UseBestOff bool
|
||||
LazyInsert uint
|
||||
|
||||
// Output
|
||||
MLen uint
|
||||
MOff uint
|
||||
Look uint
|
||||
BChar int
|
||||
BestOff [cSWD_BEST_OFF]uint
|
||||
|
||||
// Semi-public
|
||||
ctx *compressor
|
||||
mpos uint
|
||||
bestPos [cSWD_BEST_OFF]uint
|
||||
|
||||
// Private
|
||||
ip uint // input pointer (lookahead)
|
||||
bp uint // buffer pointer
|
||||
rp uint // remove pointer
|
||||
bsize uint
|
||||
bwrap []byte
|
||||
nodecount uint
|
||||
firstrp uint
|
||||
|
||||
b [cSWD_N + cSWD_F + cSWD_F]byte
|
||||
head3 [cSWD_HSIZE]uint16
|
||||
succ3 [cSWD_N + cSWD_F]uint16
|
||||
best3 [cSWD_N + cSWD_F]uint16
|
||||
llen3 [cSWD_HSIZE]uint16
|
||||
head2 [65536]uint16
|
||||
}
|
||||
|
||||
func head2(data []byte) uint {
|
||||
return uint(data[1])<<8 | uint(data[0])
|
||||
}
|
||||
|
||||
func head3(data []byte) uint {
|
||||
key := uint(data[0])
|
||||
key = (key << 5) ^ uint(data[1])
|
||||
key = (key << 5) ^ uint(data[2])
|
||||
key = (key * 0x9f5f) >> 5
|
||||
return key & (cSWD_HSIZE - 1)
|
||||
}
|
||||
|
||||
func (s *swd) gethead3(key uint) uint16 {
|
||||
if s.llen3[key] == 0 {
|
||||
return 0xFFFF
|
||||
}
|
||||
return s.head3[key]
|
||||
}
|
||||
|
||||
func (s *swd) removeNode(node uint) {
|
||||
if s.nodecount == 0 {
|
||||
key := head3(s.b[node:])
|
||||
if s.llen3[key] == 0 {
|
||||
panic("assert: swd.removeNode: invalid llen3")
|
||||
}
|
||||
s.llen3[key]--
|
||||
|
||||
key = head2(s.b[node:])
|
||||
if s.head2[key] == 0xFFFF {
|
||||
panic("assert: swd.removeNode: invalid head2")
|
||||
}
|
||||
if uint(s.head2[key]) == node {
|
||||
s.head2[key] = 0xFFFF
|
||||
}
|
||||
return
|
||||
}
|
||||
s.nodecount--
|
||||
}
|
||||
|
||||
func (s *swd) init() {
|
||||
s.SwdN = cSWD_N
|
||||
s.SwdF = cSWD_F
|
||||
s.SwdThreshold = cSWD_THRESHOLD
|
||||
|
||||
s.MaxChain = cSWD_MAX_CHAIN
|
||||
s.NiceLength = s.SwdF
|
||||
s.bsize = s.SwdN + s.SwdF
|
||||
s.bwrap = s.b[s.bsize:]
|
||||
s.nodecount = s.SwdN
|
||||
|
||||
for i := 0; i < len(s.head2); i++ {
|
||||
s.head2[i] = 0xFFFF
|
||||
}
|
||||
|
||||
s.ip = 0
|
||||
s.bp = s.ip
|
||||
s.firstrp = s.ip
|
||||
if s.ip+s.SwdF > s.bsize {
|
||||
panic("assert: swd.init: invalid ip")
|
||||
}
|
||||
|
||||
s.Look = uint(len(s.ctx.in)) - s.ip
|
||||
if s.Look > 0 {
|
||||
if s.Look > s.SwdF {
|
||||
s.Look = s.SwdF
|
||||
}
|
||||
copy(s.b[s.ip:], s.ctx.in[:s.Look])
|
||||
s.ctx.ip += int(s.Look)
|
||||
s.ip += s.Look
|
||||
}
|
||||
|
||||
if s.ip == s.bsize {
|
||||
s.ip = 0
|
||||
}
|
||||
|
||||
s.rp = s.firstrp
|
||||
if s.rp >= s.nodecount {
|
||||
s.rp -= s.nodecount
|
||||
} else {
|
||||
s.rp += s.bsize - s.nodecount
|
||||
}
|
||||
|
||||
if s.Look < 3 {
|
||||
s.b[s.bp+s.Look] = 0
|
||||
s.b[s.bp+s.Look+1] = 0
|
||||
s.b[s.bp+s.Look+2] = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (s *swd) getbyte() {
|
||||
c := -1
|
||||
if s.ctx.ip < len(s.ctx.in) {
|
||||
c = int(s.ctx.in[s.ctx.ip])
|
||||
s.ctx.ip++
|
||||
s.b[s.ip] = byte(c)
|
||||
if s.ip < s.SwdF {
|
||||
s.bwrap[s.ip] = byte(c)
|
||||
}
|
||||
} else {
|
||||
if s.Look > 0 {
|
||||
s.Look--
|
||||
}
|
||||
s.b[s.ip] = 0
|
||||
if s.ip < s.SwdF {
|
||||
s.bwrap[s.ip] = 0
|
||||
}
|
||||
}
|
||||
|
||||
s.ip++
|
||||
if s.ip == s.bsize {
|
||||
s.ip = 0
|
||||
}
|
||||
s.bp++
|
||||
if s.bp == s.bsize {
|
||||
s.bp = 0
|
||||
}
|
||||
s.rp++
|
||||
if s.rp == s.bsize {
|
||||
s.rp = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (s *swd) accept(n uint) {
|
||||
if n > s.Look {
|
||||
panic("swd: accept: invalid n")
|
||||
}
|
||||
|
||||
for i := uint(0); i < n; i++ {
|
||||
s.removeNode(s.rp)
|
||||
|
||||
key := head3(s.b[s.bp:])
|
||||
s.succ3[s.bp] = s.gethead3(key)
|
||||
s.head3[key] = uint16(s.bp)
|
||||
s.best3[s.bp] = uint16(s.SwdF + 1)
|
||||
s.llen3[key]++
|
||||
if uint(s.llen3[key]) > s.SwdN {
|
||||
panic("swd: accept: invalid llen3")
|
||||
}
|
||||
|
||||
key = head2(s.b[s.bp:])
|
||||
s.head2[key] = uint16(s.bp)
|
||||
|
||||
s.getbyte()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *swd) search(node uint, cnt uint) {
|
||||
if s.MLen <= 0 {
|
||||
panic("assert: search: invalid mlen")
|
||||
}
|
||||
|
||||
mlen := s.MLen
|
||||
bp := s.bp
|
||||
bx := s.bp + s.Look
|
||||
|
||||
scanend1 := s.b[s.bp+mlen-1]
|
||||
for ; cnt > 0; cnt-- {
|
||||
p1 := bp
|
||||
p2 := node
|
||||
px := bx
|
||||
|
||||
if mlen >= s.Look {
|
||||
panic("assert: search: invalid mlen in loop")
|
||||
}
|
||||
if s.b[p2+mlen-1] == scanend1 &&
|
||||
s.b[p2+mlen] == s.b[p1+mlen] &&
|
||||
s.b[p2] == s.b[p1] &&
|
||||
s.b[p2+1] == s.b[p1+1] {
|
||||
|
||||
if s.b[bp] != s.b[node] || s.b[bp+1] != s.b[node+1] || s.b[bp+2] != s.b[node+2] {
|
||||
panic("assert: seach: invalid initial match")
|
||||
}
|
||||
p1 = p1 + 2
|
||||
p2 = p2 + 2
|
||||
for p1 < px {
|
||||
p1++
|
||||
p2++
|
||||
if s.b[p1] != s.b[p2] {
|
||||
break
|
||||
}
|
||||
}
|
||||
i := p1 - bp
|
||||
|
||||
for j := uint(0); j < i; j++ {
|
||||
if s.b[s.bp+j] != s.b[node+j] {
|
||||
panic("assert: search: invalid final match")
|
||||
}
|
||||
}
|
||||
|
||||
if i < cSWD_BEST_OFF {
|
||||
if s.bestPos[i] == 0 {
|
||||
s.bestPos[i] = node + 1
|
||||
}
|
||||
}
|
||||
if i > mlen {
|
||||
mlen = i
|
||||
s.MLen = mlen
|
||||
s.mpos = node
|
||||
if mlen == s.Look {
|
||||
return
|
||||
}
|
||||
if mlen >= s.NiceLength {
|
||||
return
|
||||
}
|
||||
if mlen > uint(s.best3[node]) {
|
||||
return
|
||||
}
|
||||
scanend1 = s.b[s.bp+mlen-1]
|
||||
}
|
||||
}
|
||||
|
||||
node = uint(s.succ3[node])
|
||||
}
|
||||
}
|
||||
|
||||
func (s *swd) search2() bool {
|
||||
if s.Look < 2 {
|
||||
panic("assert: search2: invalid look")
|
||||
}
|
||||
if s.MLen <= 0 {
|
||||
panic("assert: search2: invalid mlen")
|
||||
}
|
||||
|
||||
key := s.head2[head2(s.b[s.bp:])]
|
||||
if key == 0xFFFF {
|
||||
return false
|
||||
}
|
||||
if s.b[s.bp] != s.b[key] || s.b[s.bp+1] != s.b[key+1] {
|
||||
panic("assert: search2: invalid key found")
|
||||
}
|
||||
if s.bestPos[2] == 0 {
|
||||
s.bestPos[2] = uint(key + 1)
|
||||
}
|
||||
if s.MLen < 2 {
|
||||
s.MLen = 2
|
||||
s.mpos = uint(key)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *swd) findbest() {
|
||||
if s.MLen == 0 {
|
||||
panic("swd: findbest: invalid mlen")
|
||||
}
|
||||
|
||||
key := head3(s.b[s.bp:])
|
||||
node := s.gethead3(key)
|
||||
s.succ3[s.bp] = node
|
||||
cnt := uint(s.llen3[key])
|
||||
s.llen3[key]++
|
||||
if cnt > s.SwdN+s.SwdF {
|
||||
panic("swd: findbest: invalid llen3")
|
||||
}
|
||||
if cnt > s.MaxChain && s.MaxChain > 0 {
|
||||
cnt = s.MaxChain
|
||||
}
|
||||
s.head3[key] = uint16(s.bp)
|
||||
|
||||
s.BChar = int(s.b[s.bp])
|
||||
len := s.MLen
|
||||
if s.MLen >= s.Look {
|
||||
if s.Look == 0 {
|
||||
s.BChar = -1
|
||||
}
|
||||
s.MOff = 0
|
||||
s.best3[s.bp] = uint16(s.SwdF + 1)
|
||||
} else {
|
||||
if s.search2() && s.Look >= 3 {
|
||||
s.search(uint(node), cnt)
|
||||
}
|
||||
|
||||
if s.MLen > len {
|
||||
s.MOff = s.pos2off(s.mpos)
|
||||
}
|
||||
|
||||
if s.UseBestOff {
|
||||
for i := 2; i < cSWD_BEST_OFF; i++ {
|
||||
if s.bestPos[i] > 0 {
|
||||
s.BestOff[i] = s.pos2off(s.bestPos[i] - 1)
|
||||
} else {
|
||||
s.BestOff[i] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
s.removeNode(s.rp)
|
||||
key = head2(s.b[s.bp:])
|
||||
s.head2[key] = uint16(s.bp)
|
||||
}
|
||||
|
||||
func (s *swd) pos2off(pos uint) uint {
|
||||
if s.bp > pos {
|
||||
return s.bp - pos
|
||||
}
|
||||
return s.bsize - (pos - s.bp)
|
||||
}
|
||||
Reference in New Issue
Block a user