diff --git a/.gitignore b/.gitignore index c6d18c7..459af73 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ None .directory .python-version .tox/ + +.DS_Store +/.vscode/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1118ca3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/ext/uchardet"] + path = src/ext/uchardet + url = https://github.com/PyYoshi/uchardet.git diff --git a/.travis.yml b/.travis.yml index c06cdbd..28f6e53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ language: python cache: directories: - $HOME/.cache/pip +git: + submodules: false matrix: include: @@ -19,6 +21,10 @@ matrix: allow_failures: - python: nightly +before_install: + - sed -i 's/git@github.com:/https:\/\/github.com\//' .gitmodules + - git submodule update --init --recursive + install: - pip install -U tox nose cython diff --git a/CHANGES.rst b/CHANGES.rst index 89fe42a..bba71c2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,35 @@ CHANGES ======= +2.0.0 (2017-04-06) +------------------ + +- Improve tests + +2.0a4 (2017-04-05) +------------------ + +- Update uchardet repo (Fix buffer overflow) + +2.0a3 (2017-03-29) +------------------ + +- Implement UniversalDetector (like chardet) + +2.0a2 (2017-03-28) +------------------ + +- Update uchardet repo (Fix memory leak) + +2.0a1 (2017-03-28) +------------------ + +- Replace `uchardet-enhanced`_ to `uchardet`_ +- Remove Detector class + +.. _uchardet-enhanced: https://bitbucket.org/medoc/uchardet-enhanced/overview +.. _uchardet: https://github.com/PyYoshi/uchardet + 1.1.3 (2017-02-26) ------------------ diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..86461c0 --- /dev/null +++ b/COPYING @@ -0,0 +1,1316 @@ + MOZILLA PUBLIC LICENSE + Version 1.1 + + --------------- + +1. Definitions. + + 1.0.1. "Commercial Use" means distribution or otherwise making the + Covered Code available to a third party. + + 1.1. "Contributor" means each entity that creates or contributes to + the creation of Modifications. + + 1.2. "Contributor Version" means the combination of the Original + Code, prior Modifications used by a Contributor, and the Modifications + made by that particular Contributor. + + 1.3. "Covered Code" means the Original Code or Modifications or the + combination of the Original Code and Modifications, in each case + including portions thereof. + + 1.4. "Electronic Distribution Mechanism" means a mechanism generally + accepted in the software development community for the electronic + transfer of data. + + 1.5. "Executable" means Covered Code in any form other than Source + Code. + + 1.6. "Initial Developer" means the individual or entity identified + as the Initial Developer in the Source Code notice required by Exhibit + A. + + 1.7. "Larger Work" means a work which combines Covered Code or + portions thereof with code not governed by the terms of this License. + + 1.8. "License" means this document. + + 1.8.1. "Licensable" means having the right to grant, to the maximum + extent possible, whether at the time of the initial grant or + subsequently acquired, any and all of the rights conveyed herein. + + 1.9. "Modifications" means any addition to or deletion from the + substance or structure of either the Original Code or any previous + Modifications. When Covered Code is released as a series of files, a + Modification is: + A. Any addition to or deletion from the contents of a file + containing Original Code or previous Modifications. + + B. Any new file that contains any part of the Original Code or + previous Modifications. + + 1.10. "Original Code" means Source Code of computer software code + which is described in the Source Code notice required by Exhibit A as + Original Code, and which, at the time of its release under this + License is not already Covered Code governed by this License. + + 1.10.1. "Patent Claims" means any patent claim(s), now owned or + hereafter acquired, including without limitation, method, process, + and apparatus claims, in any patent Licensable by grantor. + + 1.11. "Source Code" means the preferred form of the Covered Code for + making modifications to it, including all modules it contains, plus + any associated interface definition files, scripts used to control + compilation and installation of an Executable, or source code + differential comparisons against either the Original Code or another + well known, available Covered Code of the Contributor's choice. The + Source Code can be in a compressed or archival form, provided the + appropriate decompression or de-archiving software is widely available + for no charge. + + 1.12. "You" (or "Your") means an individual or a legal entity + exercising rights under, and complying with all of the terms of, this + License or a future version of this License issued under Section 6.1. + For legal entities, "You" includes any entity which controls, is + controlled by, or is under common control with You. For purposes of + this definition, "control" means (a) the power, direct or indirect, + to cause the direction or management of such entity, whether by + contract or otherwise, or (b) ownership of more than fifty percent + (50%) of the outstanding shares or beneficial ownership of such + entity. + +2. Source Code License. + + 2.1. The Initial Developer Grant. + The Initial Developer hereby grants You a world-wide, royalty-free, + non-exclusive license, subject to third party intellectual property + claims: + (a) under intellectual property rights (other than patent or + trademark) Licensable by Initial Developer to use, reproduce, + modify, display, perform, sublicense and distribute the Original + Code (or portions thereof) with or without Modifications, and/or + as part of a Larger Work; and + + (b) under Patents Claims infringed by the making, using or + selling of Original Code, to make, have made, use, practice, + sell, and offer for sale, and/or otherwise dispose of the + Original Code (or portions thereof). + + (c) the licenses granted in this Section 2.1(a) and (b) are + effective on the date Initial Developer first distributes + Original Code under the terms of this License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is + granted: 1) for code that You delete from the Original Code; 2) + separate from the Original Code; or 3) for infringements caused + by: i) the modification of the Original Code or ii) the + combination of the Original Code with other software or devices. + + 2.2. Contributor Grant. + Subject to third party intellectual property claims, each Contributor + hereby grants You a world-wide, royalty-free, non-exclusive license + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Contributor, to use, reproduce, modify, + display, perform, sublicense and distribute the Modifications + created by such Contributor (or portions thereof) either on an + unmodified basis, with other Modifications, as Covered Code + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or + selling of Modifications made by that Contributor either alone + and/or in combination with its Contributor Version (or portions + of such combination), to make, use, sell, offer for sale, have + made, and/or otherwise dispose of: 1) Modifications made by that + Contributor (or portions thereof); and 2) the combination of + Modifications made by that Contributor with its Contributor + Version (or portions of such combination). + + (c) the licenses granted in Sections 2.2(a) and 2.2(b) are + effective on the date Contributor first makes Commercial Use of + the Covered Code. + + (d) Notwithstanding Section 2.2(b) above, no patent license is + granted: 1) for any code that Contributor has deleted from the + Contributor Version; 2) separate from the Contributor Version; + 3) for infringements caused by: i) third party modifications of + Contributor Version or ii) the combination of Modifications made + by that Contributor with other software (except as part of the + Contributor Version) or other devices; or 4) under Patent Claims + infringed by Covered Code in the absence of Modifications made by + that Contributor. + +3. Distribution Obligations. + + 3.1. Application of License. + The Modifications which You create or to which You contribute are + governed by the terms of this License, including without limitation + Section 2.2. The Source Code version of Covered Code may be + distributed only under the terms of this License or a future version + of this License released under Section 6.1, and You must include a + copy of this License with every copy of the Source Code You + distribute. You may not offer or impose any terms on any Source Code + version that alters or restricts the applicable version of this + License or the recipients' rights hereunder. However, You may include + an additional document offering the additional rights described in + Section 3.5. + + 3.2. Availability of Source Code. + Any Modification which You create or to which You contribute must be + made available in Source Code form under the terms of this License + either on the same media as an Executable version or via an accepted + Electronic Distribution Mechanism to anyone to whom you made an + Executable version available; and if made available via Electronic + Distribution Mechanism, must remain available for at least twelve (12) + months after the date it initially became available, or at least six + (6) months after a subsequent version of that particular Modification + has been made available to such recipients. You are responsible for + ensuring that the Source Code version remains available even if the + Electronic Distribution Mechanism is maintained by a third party. + + 3.3. Description of Modifications. + You must cause all Covered Code to which You contribute to contain a + file documenting the changes You made to create that Covered Code and + the date of any change. You must include a prominent statement that + the Modification is derived, directly or indirectly, from Original + Code provided by the Initial Developer and including the name of the + Initial Developer in (a) the Source Code, and (b) in any notice in an + Executable version or related documentation in which You describe the + origin or ownership of the Covered Code. + + 3.4. Intellectual Property Matters + (a) Third Party Claims. + If Contributor has knowledge that a license under a third party's + intellectual property rights is required to exercise the rights + granted by such Contributor under Sections 2.1 or 2.2, + Contributor must include a text file with the Source Code + distribution titled "LEGAL" which describes the claim and the + party making the claim in sufficient detail that a recipient will + know whom to contact. If Contributor obtains such knowledge after + the Modification is made available as described in Section 3.2, + Contributor shall promptly modify the LEGAL file in all copies + Contributor makes available thereafter and shall take other steps + (such as notifying appropriate mailing lists or newsgroups) + reasonably calculated to inform those who received the Covered + Code that new knowledge has been obtained. + + (b) Contributor APIs. + If Contributor's Modifications include an application programming + interface and Contributor has knowledge of patent licenses which + are reasonably necessary to implement that API, Contributor must + also include this information in the LEGAL file. + + (c) Representations. + Contributor represents that, except as disclosed pursuant to + Section 3.4(a) above, Contributor believes that Contributor's + Modifications are Contributor's original creation(s) and/or + Contributor has sufficient rights to grant the rights conveyed by + this License. + + 3.5. Required Notices. + You must duplicate the notice in Exhibit A in each file of the Source + Code. If it is not possible to put such notice in a particular Source + Code file due to its structure, then You must include such notice in a + location (such as a relevant directory) where a user would be likely + to look for such a notice. If You created one or more Modification(s) + You may add your name as a Contributor to the notice described in + Exhibit A. You must also duplicate this License in any documentation + for the Source Code where You describe recipients' rights or ownership + rights relating to Covered Code. You may choose to offer, and to + charge a fee for, warranty, support, indemnity or liability + obligations to one or more recipients of Covered Code. However, You + may do so only on Your own behalf, and not on behalf of the Initial + Developer or any Contributor. You must make it absolutely clear than + any such warranty, support, indemnity or liability obligation is + offered by You alone, and You hereby agree to indemnify the Initial + Developer and every Contributor for any liability incurred by the + Initial Developer or such Contributor as a result of warranty, + support, indemnity or liability terms You offer. + + 3.6. Distribution of Executable Versions. + You may distribute Covered Code in Executable form only if the + requirements of Section 3.1-3.5 have been met for that Covered Code, + and if You include a notice stating that the Source Code version of + the Covered Code is available under the terms of this License, + including a description of how and where You have fulfilled the + obligations of Section 3.2. The notice must be conspicuously included + in any notice in an Executable version, related documentation or + collateral in which You describe recipients' rights relating to the + Covered Code. You may distribute the Executable version of Covered + Code or ownership rights under a license of Your choice, which may + contain terms different from this License, provided that You are in + compliance with the terms of this License and that the license for the + Executable version does not attempt to limit or alter the recipient's + rights in the Source Code version from the rights set forth in this + License. If You distribute the Executable version under a different + license You must make it absolutely clear that any terms which differ + from this License are offered by You alone, not by the Initial + Developer or any Contributor. You hereby agree to indemnify the + Initial Developer and every Contributor for any liability incurred by + the Initial Developer or such Contributor as a result of any such + terms You offer. + + 3.7. Larger Works. + You may create a Larger Work by combining Covered Code with other code + not governed by the terms of this License and distribute the Larger + Work as a single product. In such a case, You must make sure the + requirements of this License are fulfilled for the Covered Code. + +4. Inability to Comply Due to Statute or Regulation. + + If it is impossible for You to comply with any of the terms of this + License with respect to some or all of the Covered Code due to + statute, judicial order, or regulation then You must: (a) comply with + the terms of this License to the maximum extent possible; and (b) + describe the limitations and the code they affect. Such description + must be included in the LEGAL file described in Section 3.4 and must + be included with all distributions of the Source Code. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Application of this License. + + This License applies to code to which the Initial Developer has + attached the notice in Exhibit A and to related Covered Code. + +6. Versions of the License. + + 6.1. New Versions. + Netscape Communications Corporation ("Netscape") may publish revised + and/or new versions of the License from time to time. Each version + will be given a distinguishing version number. + + 6.2. Effect of New Versions. + Once Covered Code has been published under a particular version of the + License, You may always continue to use it under the terms of that + version. You may also choose to use such Covered Code under the terms + of any subsequent version of the License published by Netscape. No one + other than Netscape has the right to modify the terms applicable to + Covered Code created under this License. + + 6.3. Derivative Works. + If You create or use a modified version of this License (which you may + only do in order to apply it to code which is not already Covered Code + governed by this License), You must (a) rename Your license so that + the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", + "MPL", "NPL" or any confusingly similar phrase do not appear in your + license (except to note that your license differs from this License) + and (b) otherwise make it clear that Your version of the license + contains terms which differ from the Mozilla Public License and + Netscape Public License. (Filling in the name of the Initial + Developer, Original Code or Contributor in the notice described in + Exhibit A shall not of themselves be deemed to be modifications of + this License.) + +7. DISCLAIMER OF WARRANTY. + + COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, + WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF + DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. + THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE + IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, + YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE + COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER + OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF + ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +8. TERMINATION. + + 8.1. This License and the rights granted hereunder will terminate + automatically if You fail to comply with terms herein and fail to cure + such breach within 30 days of becoming aware of the breach. All + sublicenses to the Covered Code which are properly granted shall + survive any termination of this License. Provisions which, by their + nature, must remain in effect beyond the termination of this License + shall survive. + + 8.2. If You initiate litigation by asserting a patent infringement + claim (excluding declatory judgment actions) against Initial Developer + or a Contributor (the Initial Developer or Contributor against whom + You file such action is referred to as "Participant") alleging that: + + (a) such Participant's Contributor Version directly or indirectly + infringes any patent, then any and all rights granted by such + Participant to You under Sections 2.1 and/or 2.2 of this License + shall, upon 60 days notice from Participant terminate prospectively, + unless if within 60 days after receipt of notice You either: (i) + agree in writing to pay Participant a mutually agreeable reasonable + royalty for Your past and future use of Modifications made by such + Participant, or (ii) withdraw Your litigation claim with respect to + the Contributor Version against such Participant. If within 60 days + of notice, a reasonable royalty and payment arrangement are not + mutually agreed upon in writing by the parties or the litigation claim + is not withdrawn, the rights granted by Participant to You under + Sections 2.1 and/or 2.2 automatically terminate at the expiration of + the 60 day notice period specified above. + + (b) any software, hardware, or device, other than such Participant's + Contributor Version, directly or indirectly infringes any patent, then + any rights granted to You by such Participant under Sections 2.1(b) + and 2.2(b) are revoked effective as of the date You first made, used, + sold, distributed, or had made, Modifications made by that + Participant. + + 8.3. If You assert a patent infringement claim against Participant + alleging that such Participant's Contributor Version directly or + indirectly infringes any patent where such claim is resolved (such as + by license or settlement) prior to the initiation of patent + infringement litigation, then the reasonable value of the licenses + granted by such Participant under Sections 2.1 or 2.2 shall be taken + into account in determining the amount or value of any payment or + license. + + 8.4. In the event of termination under Sections 8.1 or 8.2 above, + all end user license agreements (excluding distributors and resellers) + which have been validly granted by You or any distributor hereunder + prior to termination shall survive termination. + +9. LIMITATION OF LIABILITY. + + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL + DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, + OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR + ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY + CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, + WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER + COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN + INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF + LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY + RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW + PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE + EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO + THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +10. U.S. GOVERNMENT END USERS. + + The Covered Code is a "commercial item," as that term is defined in + 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer + software" and "commercial computer software documentation," as such + terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 + C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), + all U.S. Government End Users acquire Covered Code with only those + rights set forth herein. + +11. MISCELLANEOUS. + + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. This License shall be governed by + California law provisions (except to the extent applicable law, if + any, provides otherwise), excluding its conflict-of-law provisions. + With respect to disputes in which at least one party is a citizen of, + or an entity chartered or registered to do business in the United + States of America, any litigation relating to this License shall be + subject to the jurisdiction of the Federal Courts of the Northern + District of California, with venue lying in Santa Clara County, + California, with the losing party responsible for costs, including + without limitation, court costs and reasonable attorneys' fees and + expenses. The application of the United Nations Convention on + Contracts for the International Sale of Goods is expressly excluded. + Any law or regulation which provides that the language of a contract + shall be construed against the drafter shall not apply to this + License. + +12. RESPONSIBILITY FOR CLAIMS. + + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or indirectly, + out of its utilization of rights under this License and You agree to + work with Initial Developer and Contributors to distribute such + responsibility on an equitable basis. Nothing herein is intended or + shall be deemed to constitute any admission of liability. + +13. MULTIPLE-LICENSED CODE. + + Initial Developer may designate portions of the Covered Code as + "Multiple-Licensed". "Multiple-Licensed" means that the Initial + Developer permits you to utilize portions of the Covered Code under + Your choice of the NPL or the alternative licenses, if any, specified + by the Initial Developer in the file described in Exhibit A. + +EXHIBIT A -Mozilla Public License. + + ``The contents of this file are subject to the Mozilla Public License + Version 1.1 (the "License"); you may not use this file except in + compliance with the License. You may obtain a copy of the License at + http://www.mozilla.org/MPL/ + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + License for the specific language governing rights and limitations + under the License. + + The Original Code is ______________________________________. + + The Initial Developer of the Original Code is ________________________. + Portions created by ______________________ are Copyright (C) ______ + _______________________. All Rights Reserved. + + Contributor(s): ______________________________________. + + Alternatively, the contents of this file may be used under the terms + of the _____ license (the "[___] License"), in which case the + provisions of [______] License are applicable instead of those + above. If you wish to allow use of your version of this file only + under the terms of the [____] License and not to allow others to use + your version of this file under the MPL, indicate your decision by + deleting the provisions above and replace them with the notice and + other provisions required by the [___] License. If you do not delete + the provisions above, a recipient may use your version of this file + under either the MPL or the [___] License." + + [NOTE: The text of this Exhibit A may differ slightly from the text of + the notices in the Source Code files of the Original Code. You should + use the text of this Exhibit A rather than the text found in the + Original Code Source Code for Your Modifications.] + +------------------------------------------------------------------------ + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. + +------------------------------------------------------------------------ + + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/Makefile b/Makefile index abd0d6f..e695467 100644 --- a/Makefile +++ b/Makefile @@ -5,19 +5,20 @@ clean: $(RM) -r build dist src/cchardet/__pycache__ src/cchardet/*.cpp src/cchardet/*.pyc src/cchardet/*.so src/cchardet.egg-info src/tests/__pycache__ src/tests/*.pyc sdist: - python setup.py sdist + python setup.py sdist --formats=gztar pip: pip install -U pip cython tox nose chardet twine: - twine upload dist/*.whl - twine upload dist/*.tar.gz + twine upload dist/cchardet-*.whl dist/cchardet-*.tar.gz install: clean python setup.py install build-manylinux1-wheel: + docker pull quay.io/pypa/manylinux1_i686 + docker pull quay.io/pypa/manylinux1_x86_64 docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686 bash dockerfiles/buildwheel.sh docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash dockerfiles/buildwheel.sh diff --git a/README.rst b/README.rst index 79fba94..8e26fb8 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ cChardet ======== -cChardet is high speed universal character encoding detector. - binding to `charsetdetect`_. +cChardet is high speed universal character encoding detector. - binding to `uchardet`_. .. image:: https://badge.fury.io/py/cchardet.svg :target: https://badge.fury.io/py/cchardet @@ -13,45 +13,195 @@ cChardet is high speed universal character encoding detector. - binding to `char :target: https://ci.appveyor.com/project/PyYoshi/cchardet/branch/master :alt: AppVeyor build status -Support codecs --------------- +Supported Languages/Encodings +----------------------------- -- Big5 -- EUC-JP -- EUC-KR -- GB18030 -- HZ-GB-2312 -- IBM855 -- IBM866 -- ISO-2022-CN -- ISO-2022-JP -- ISO-2022-KR -- ISO-8859-2 -- ISO-8859-5 -- ISO-8859-7 -- ISO-8859-8 -- KOI8-R -- Shift_JIS -- TIS-620 -- UTF-8 -- UTF-16BE -- UTF-16LE -- UTF-32BE -- UTF-32LE -- WINDOWS-1250 -- WINDOWS-1251 -- WINDOWS-1252 -- WINDOWS-1253 -- WINDOWS-1255 -- EUC-TW -- X-ISO-10646-UCS-4-2143 -- X-ISO-10646-UCS-4-3412 -- x-mac-cyrillic +- International (Unicode) -Requirements ------------- + - UTF-8 + - UTF-16BE / UTF-16LE + - UTF-32BE / UTF-32LE / X-ISO-10646-UCS-4-34121 / + X-ISO-10646-UCS-4-21431 -- `Cython`_ +- Arabic + + - ISO-8859-6 + - WINDOWS-1256 + +- Bulgarian + + - ISO-8859-5 + - WINDOWS-1251 + +- Chinese + + - ISO-2022-CN + - BIG5 + - EUC-TW + - GB18030 + - HZ-GB-2312 + +- Croatian: + + - ISO-8859-2 + - ISO-8859-13 + - ISO-8859-16 + - Windows-1250 + - IBM852 + - MAC-CENTRALEUROPE + +- Czech + + - Windows-1250 + - ISO-8859-2 + - IBM852 + - MAC-CENTRALEUROPE + +- Danish + + - ISO-8859-1 + - ISO-8859-15 + - WINDOWS-1252 + +- English + + - ASCII + +- Esperanto + + - ISO-8859-3 + +- Estonian + + - ISO-8859-4 + - ISO-8859-13 + - ISO-8859-13 + - Windows-1252 + - Windows-1257 + +- Finnish + + - ISO-8859-1 + - ISO-8859-4 + - ISO-8859-9 + - ISO-8859-13 + - ISO-8859-15 + - WINDOWS-1252 + +- French + + - ISO-8859-1 + - ISO-8859-15 + - WINDOWS-1252 + +- German + + - ISO-8859-1 + - WINDOWS-1252 + +- Greek + + - ISO-8859-7 + - WINDOWS-1253 + +- Hebrew + + - ISO-8859-8 + - WINDOWS-1255 + +- Hungarian: + + - ISO-8859-2 + - WINDOWS-1250 + +- Irish Gaelic + + - ISO-8859-1 + - ISO-8859-9 + - ISO-8859-15 + - WINDOWS-1252 + +- Italian + + - ISO-8859-1 + - ISO-8859-3 + - ISO-8859-9 + - ISO-8859-15 + - WINDOWS-1252 + +- Japanese + + - ISO-2022-JP + - SHIFT\_JIS + - EUC-JP + +- Korean + + - ISO-2022-KR + - EUC-KR / UHC + +- Lithuanian + + - ISO-8859-4 + - ISO-8859-10 + - ISO-8859-13 + +- Latvian + + - ISO-8859-4 + - ISO-8859-10 + - ISO-8859-13 + +- Maltese + + - ISO-8859-3 + +- Polish: + + - ISO-8859-2 + - ISO-8859-13 + - ISO-8859-16 + - Windows-1250 + - IBM852 + - MAC-CENTRALEUROPE + +- Portuguese + + - ISO-8859-1 + - ISO-8859-9 + - ISO-8859-15 + - WINDOWS-1252 + +- Romanian: + + - ISO-8859-2 + - ISO-8859-16 + - Windows-1250 + - IBM852 + +- Russian + + - ISO-8859-5 + - KOI8-R + - WINDOWS-1251 + - MAC-CYRILLIC + - IBM866 + - IBM855 + +- Slovak + + - Windows-1250 + - ISO-8859-2 + - IBM852 + - MAC-CENTRALEUROPE + +- Slovene + + - ISO-8859-2 + - ISO-8859-16 + - Windows-1250 + - IBM852 + - M Example ------- @@ -60,12 +210,11 @@ Example # -*- coding: utf-8 -*- import cchardet as chardet - with open(r"src/tests/testdata/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt", "rb") as f: + with open(r"src/tests/samples/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt", "rb") as f: msg = f.read() result = chardet.detect(msg) print(result) - Benchmark --------- @@ -79,7 +228,7 @@ Benchmark Results ~~~~~~~ -CPU: Intel(R) Core(TM) i3-4170 CPU @ 3.70GHz +CPU: Intel(R) Core(TM) i5-4690 CPU @ 3.50GHz RAM: DDR3 1600Mhz 16GB @@ -93,41 +242,31 @@ Python 2.7.12 +==========+==================+ | chardet | 0.26 | +----------+------------------+ -| cchardet | 1408.73 | +| cchardet | 1341.81 | +----------+------------------+ -Python 3.5.2 +Python 3.6.0 ^^^^^^^^^^^^ +----------+------------------+ | | Request (call/s) | +==========+==================+ -| chardet | 0.28 | +| chardet | 0.26 | +----------+------------------+ -| cchardet | 1380.40 | +| cchardet | 1472.43 | +----------+------------------+ -License + +LICENSE ------- -- The MIT License: `src/cchardet`_ -- Other Libraries License: Please, look at the `src/ext`_ directory. - -Thanks ------- - -- `uchardet-enhanced`_ -- `Cython`_ +See **COPYING** file. Contact ------- -`Issues`_ +- `Issues`_ -.. _charsetdetect: https://bitbucket.org/medoc/uchardet-enhanced/overview -.. _Cython: http://www.cython.org/ -.. _src/cchardet: https://github.com/PyYoshi/cChardet/tree/master/src/cchardet -.. _src/ext: https://github.com/PyYoshi/cChardet/tree/master/src/ext -.. _uchardet-enhanced: https://bitbucket.org/medoc/uchardet-enhanced/overview +.. _uchardet: https://github.com/PyYoshi/uchardet .. _Issues: https://github.com/PyYoshi/cChardet/issues?page=1&state=open diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..1bf4c34 --- /dev/null +++ b/TODO.md @@ -0,0 +1,2 @@ +- Implement cli tool (like chardet cli) +- Improve docs diff --git a/appveyor.yml b/appveyor.yml index d72d295..2578e04 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,6 +11,7 @@ environment: - PYTHON: "C:\\Python36-x64" install: + - "git submodule update --init --recursive" - "%PYTHON%\\python.exe -m pip install -U pip wheel nose cython" build: off diff --git a/setup.py b/setup.py index 377e8b3..1639fae 100644 --- a/setup.py +++ b/setup.py @@ -1,108 +1,143 @@ -#!/usr/bin/env python -# coding: utf-8 - -# python setup.py sdist --formats=gztar - -import os -import sys -import platform -import glob -import codecs -import re - -try: - from setuptools import setup, Extension -except ImportError: - from distutils.core import setup, Extension - -try: - import Cython.Compiler.Main as cython_compiler - - have_cython = True -except ImportError: - have_cython = False -from distutils.command.build_ext import build_ext - -DEBUG = False - -src_dir = 'src' -ext_dir = os.path.join(src_dir, 'ext') -build_dir = 'build' -cchardet_dir = os.path.join(src_dir, 'cchardet/') -charsetdetect_dir = os.path.join(ext_dir, 'libcharsetdetect/') -nspr_emu_dir = os.path.join(charsetdetect_dir, 'nspr-emu/') -uchardet_dir = os.path.join(charsetdetect_dir, 'mozilla/extensions/universalchardet/src/base/') - -if have_cython: - pyx_sources = glob.glob(cchardet_dir + '*.pyx') - sys.stderr.write('cythonize: %r\n' % (pyx_sources,)) - cython_compiler.compile(pyx_sources, options=cython_compiler.CompilationOptions(cplus=True)) -cchardet_sources = glob.glob(cchardet_dir + '*.cpp') -sources = cchardet_sources + [os.path.join(charsetdetect_dir, 'charsetdetect.cpp')] + glob.glob(uchardet_dir + '*.cpp') - -macros = [] -extra_compile_args = [] -extra_link_args = [] - -if platform.system() == 'Windows': - macros.append(('WIN32', '1')) - -if DEBUG: - macros.append(('DEBUG_chardet', '1')) - extra_compile_args.append('-g'), - extra_link_args.append('-g'), - -cchardet_module = Extension( - 'cchardet._cchardet', - sources=sources, - include_dirs=[uchardet_dir, nspr_emu_dir, charsetdetect_dir], - language='c++', - define_macros=macros, -) - - -def read(f): - return open(os.path.join(os.path.dirname(__file__), f)).read().strip() - - -with codecs.open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'src', 'cchardet', 'version.py'), 'r', - 'latin1') as fp: - try: - version = re.findall(r"^__version__ = '([^']+)'\r?$", - fp.read(), re.M)[0] - except IndexError: - raise RuntimeError('Unable to determine version.') - -setup( - name='cchardet', - author='PyYoshi', - author_email='myoshi321go@gmail.com', - url=r'https://github.com/PyYoshi/cChardet', - description='Universal encoding detector. This library is faster than chardet.', - long_description='\n\n'.join((read('README.rst'), read('CHANGES.rst'))), - version=version, - license='MIT License', - classifiers=[ - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Cython', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - ], - keywords=[ - 'cython', - 'chardet', - 'charsetdetect' - ], - cmdclass={'build_ext': build_ext}, - package_dir={'': src_dir}, - packages=['cchardet', ], - ext_modules=[ - cchardet_module - ], -) +#!/usr/bin/env python +# coding: utf-8 + +import os +import sys +import glob +import codecs +import re +from distutils.command.build_ext import build_ext +from distutils import sysconfig + +try: + from setuptools import setup, Extension +except ImportError: + from distutils.core import setup, Extension + +have_cython = True +try: + import Cython.Compiler.Main as cython_compiler +except ImportError: + have_cython = False + +cchardet_dir = 'src/cchardet/' +uchardet_dir = 'src/ext/uchardet/src' + +if have_cython: + pyx_sources = glob.glob(cchardet_dir + '*.pyx') + sys.stderr.write('cythonize: %r\n' % (pyx_sources,)) + cython_compiler.compile(pyx_sources, options=cython_compiler.CompilationOptions(cplus=True)) + +cchardet_sources = glob.glob(cchardet_dir + '*.cpp') +sources = cchardet_sources + +uchardet_sources = [ + os.path.join(uchardet_dir, 'CharDistribution.cpp'), + os.path.join(uchardet_dir, 'JpCntx.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangArabicModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangBulgarianModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangCroatianModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangCzechModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangEsperantoModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangEstonianModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangFinnishModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangFrenchModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangDanishModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangGermanModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangGreekModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangHungarianModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangHebrewModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangIrishModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangItalianModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangLithuanianModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangLatvianModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangMalteseModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangPolishModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangPortugueseModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangRomanianModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangRussianModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangSlovakModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangSloveneModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangSwedishModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangSpanishModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangThaiModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangTurkishModel.cpp'), + os.path.join(uchardet_dir, 'LangModels/LangVietnameseModel.cpp'), + os.path.join(uchardet_dir, 'nsHebrewProber.cpp'), + os.path.join(uchardet_dir, 'nsCharSetProber.cpp'), + os.path.join(uchardet_dir, 'nsBig5Prober.cpp'), + os.path.join(uchardet_dir, 'nsEUCJPProber.cpp'), + os.path.join(uchardet_dir, 'nsEUCKRProber.cpp'), + os.path.join(uchardet_dir, 'nsEUCTWProber.cpp'), + os.path.join(uchardet_dir, 'nsEscCharsetProber.cpp'), + os.path.join(uchardet_dir, 'nsEscSM.cpp'), + os.path.join(uchardet_dir, 'nsGB2312Prober.cpp'), + os.path.join(uchardet_dir, 'nsMBCSGroupProber.cpp'), + os.path.join(uchardet_dir, 'nsMBCSSM.cpp'), + os.path.join(uchardet_dir, 'nsSBCSGroupProber.cpp'), + os.path.join(uchardet_dir, 'nsSBCharSetProber.cpp'), + os.path.join(uchardet_dir, 'nsSJISProber.cpp'), + os.path.join(uchardet_dir, 'nsUTF8Prober.cpp'), + os.path.join(uchardet_dir, 'nsLatin1Prober.cpp'), + os.path.join(uchardet_dir, 'nsUniversalDetector.cpp'), + os.path.join(uchardet_dir, 'uchardet.cpp') +] +sources += uchardet_sources + +# Remove the "-Wstrict-prototypes" compiler option, which isn't valid for C++. +cfg_vars = sysconfig.get_config_vars() +for key, value in cfg_vars.items(): + if type(value) == str: + cfg_vars[key] = value.replace("-Wstrict-prototypes", "") + +cchardet_module = Extension( + 'cchardet._cchardet', + sources=sources, + include_dirs=[uchardet_dir], + language='c++', +) + +def read(f): + return open(os.path.join(os.path.dirname(__file__), f)).read().strip() + +with codecs.open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'src', 'cchardet', 'version.py'), 'r', 'latin1') as fp: + try: + version = re.findall(r"^__version__ = '([^']+)'\r?$", fp.read(), re.M)[0] + except IndexError: + raise RuntimeError('Unable to determine version.') + +setup( + name='cchardet', + author='PyYoshi', + author_email='myoshi321go@gmail.com', + url=r'https://github.com/PyYoshi/cChardet', + description='cChardet is high speed universal character encoding detector.', + long_description='\n\n'.join((read('README.rst'), read('CHANGES.rst'))), + version=version, + license='Mozilla Public License', + classifiers=[ + 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)', + 'Programming Language :: Cython', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ], + keywords=[ + 'cython', + 'chardet', + 'charsetdetect' + ], + cmdclass={'build_ext': build_ext}, + package_dir={'': 'src'}, + packages=['cchardet', ], + ext_modules=[ + cchardet_module + ], +) diff --git a/src/cchardet/__init__.py b/src/cchardet/__init__.py index 3659a4a..fa83d73 100644 --- a/src/cchardet/__init__.py +++ b/src/cchardet/__init__.py @@ -17,12 +17,19 @@ def detect(msg): encoding = encoding.decode() return {"encoding": encoding, "confidence": confidence} - -class Detector(object): - """Wrap csd_consider with 'feed' feature.""" - +class UniversalDetector(object): def __init__(self): - self._detector = _cchardet.Detector() + self._detector = _cchardet.UniversalDetector() + + def __enter__(self): + return self + + def __exit__(self, exception_type, exception_value, traceback): + self.close() + return False + + def reset(self): + self._detector.reset() def feed(self, data): self._detector.feed(data) diff --git a/src/cchardet/_cchardet.pyx b/src/cchardet/_cchardet.pyx index f03f1f9..690d5f3 100644 --- a/src/cchardet/_cchardet.pyx +++ b/src/cchardet/_cchardet.pyx @@ -1,73 +1,87 @@ cdef extern from *: ctypedef char* const_char_ptr "const char*" -cdef extern from "charsetdetect.h": - ctypedef void* csd_t - cdef csd_t csd_open() - cdef int csd_consider(csd_t csd, char* data, int length) - cdef const_char_ptr csd_close2(csd_t csd, float *confidence) +cdef extern from "uchardet.h": + ctypedef void* uchardet_t + cdef uchardet_t uchardet_new() + cdef void uchardet_delete(uchardet_t ud) + cdef int uchardet_handle_data(uchardet_t ud, const_char_ptr data, int length) + cdef void uchardet_data_end(uchardet_t ud) + cdef void uchardet_reset(uchardet_t ud) + cdef const_char_ptr uchardet_get_charset(uchardet_t ud) + cdef float uchardet_get_confidence(uchardet_t ud) -def detect_with_confidence(char *msg): - cdef csd_t csd = csd_open() - - # すでにカウント済みの長さへアクセス - # strlenでは再度カウントすることになる - # https://github.com/python/cpython/blob/c30098c8c6014f3340a369a31df9c74bdbacc269/Include/bytesobject.h#L82 - # https://github.com/python/cpython/blob/c30098c8c6014f3340a369a31df9c74bdbacc269/Objects/bytesobject.c#L2490 - # https://github.com/python/cpython/blob/c30098c8c6014f3340a369a31df9c74bdbacc269/Include/object.h#L346 - # https://github.com/python/cpython/blob/c30098c8c6014f3340a369a31df9c74bdbacc269/Objects/bytesobject.c#L2410 +def detect_with_confidence(const_char_ptr msg): cdef int length = len(msg) + + cdef uchardet_t ud = uchardet_new() - cdef int result = csd_consider(csd, msg, length) - cdef float confidence = 0.0 - cdef const_char_ptr detected_charset + cdef int result = uchardet_handle_data(ud, msg, length) + if result == -1: + uchardet_delete(ud) + raise Exception("Handle data error") - if result == 1: # Need more data - detected_charset = csd_close2(csd, &confidence) - elif result == 0: # Detected early - detected_charset = csd_close2(csd, &confidence) - else: # Error, signal with a negative number - raise Exception("Error, signal with a negative number") + uchardet_data_end(ud) + + cdef bytes detected_charset = uchardet_get_charset(ud) + cdef float detected_confidence = uchardet_get_confidence(ud) + uchardet_delete(ud) if detected_charset: - return detected_charset, confidence + return detected_charset, detected_confidence + return None, None -cdef class Detector: - cdef csd_t csd +cdef class UniversalDetector: + cdef uchardet_t _ud cdef int _done cdef int _closed - cdef float _confidence - cdef const_char_ptr _detected_charset + cdef bytes _detected_charset + cdef float _detected_confidence def __init__(self): - self.csd = csd_open() + self._ud = uchardet_new() self._done = 0 self._closed = 0 - self._confidence = 0.0 - self._detected_charset = '' + self._detected_charset = b"" + self._detected_confidence = 0.0 - def feed(self, char *msg): + def reset(self): + if not self._closed: + self._done = 0 + self._closed = 0 + self._detected_charset = b"" + self._detected_confidence = 0.0 + uchardet_reset(self._ud) + + def feed(self, const_char_ptr msg): cdef int length cdef int result - if not self.done and not self._closed: - length = len(msg) - result = csd_consider(self.csd, msg, length) + if self._closed: + return - if result == -1: # Error, signal with a negative number - raise Exception("Error, signal with a negative number") + length = len(msg) + if length > 0: + result = uchardet_handle_data(self._ud, msg, length) - elif result == 1: # Need more data - pass - - elif result == 0: # Detected early + if result == -1: + self._closed = 1 + uchardet_delete(self._ud) + raise Exception("Handle data error") + elif result == 0: self._done = 1 - self.close() + + self._detected_charset = uchardet_get_charset(self._ud) + self._detected_confidence = uchardet_get_confidence(self._ud) def close(self): if not self._closed: - self._detected_charset = csd_close2(self.csd, &self._confidence) + uchardet_data_end(self._ud) + self._detected_charset = uchardet_get_charset(self._ud) + self._detected_confidence = uchardet_get_confidence(self._ud) + + uchardet_delete(self._ud) self._closed = 1 @property @@ -77,5 +91,6 @@ cdef class Detector: @property def result(self): if len(self._detected_charset): - return self._detected_charset, self._confidence - return None, None + return self._detected_charset, self._detected_confidence + else: + return None, None diff --git a/src/cchardet/version.py b/src/cchardet/version.py index 7bb021e..afced14 100644 --- a/src/cchardet/version.py +++ b/src/cchardet/version.py @@ -1 +1 @@ -__version__ = '1.1.3' +__version__ = '2.0.0' diff --git a/src/ext/libcharsetdetect.txt b/src/ext/libcharsetdetect.txt deleted file mode 100644 index bd921d8..0000000 --- a/src/ext/libcharsetdetect.txt +++ /dev/null @@ -1,18 +0,0 @@ -Overview: - -This package implements improvements forr the mozilla universalchardet -module described in: - http://www-archive.mozilla.org/projects/intl/UniversalCharsetDetection.html - -The interface uses the C wrapper described here: - https://github.com/batterseapower/libcharsetdetect - -Modifications / improvements affect the core universalchardet module, not -the C wrapper (which is useful and necessary anyway for integration and -testing) - -Directory contents: - -libcharsetdetect/ - - The C API from the reference above, with the modified mozilla code - inside libcharsetdetect/mozilla/extensions/universalchardet/src/base/ diff --git a/src/ext/libcharsetdetect/.gitignore b/src/ext/libcharsetdetect/.gitignore deleted file mode 100644 index 2a0df09..0000000 --- a/src/ext/libcharsetdetect/.gitignore +++ /dev/null @@ -1,28 +0,0 @@ -# OS junk -.DS_Store -Thumbs.db - -# XCode build artifacts -build/ - -# XCode user configuration -*.mode1v3 -*.pbxuser - -# XCode something or other.. not sure if it should be checked in -charsetdetectProj.xcconfig -charsetdetectTarget.xcconfig - -# CMake generated build system -Makefile - -# CMake build artifacts -lib/ - -# CMake junk -CMakeCache.txt -CMakeFiles/ -cmake_install.cmake -install_manifest.txt -*.o -*.def \ No newline at end of file diff --git a/src/ext/libcharsetdetect/CMakeLists.txt b/src/ext/libcharsetdetect/CMakeLists.txt deleted file mode 100644 index 0d3926f..0000000 --- a/src/ext/libcharsetdetect/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) - -#IF("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -std=c++0x" ) -#ENDIF("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") - -# Add a distclean target to the generated makefile -IF (UNIX) - ADD_CUSTOM_TARGET (distclean @echo cleaning for source distribution) - SET(DISTCLEANED - cmake.depends - cmake.check_depends - CMakeCache.txt - cmake.check_cache - *.cmake - Makefile - core core.* - gmon.out - *~ - ) - - ADD_CUSTOM_COMMAND( - DEPENDS clean - COMMENT "distribution clean" - COMMAND rm - ARGS -Rf CMakeTmp ${DISTCLEANED} - TARGET distclean - ) -ENDIF(UNIX) - - -PROJECT(charsetdetect) - -INCLUDE_DIRECTORIES(. nspr-emu mozilla/extensions/universalchardet/src/base/) - -FILE(GLOB charsetdetect_lib_src mozilla/extensions/universalchardet/src/base/*.cpp) -SET(charsetdetect_lib_src ${charsetdetect_lib_src} charsetdetect.cpp) -ADD_LIBRARY(charsetdetect ${charsetdetect_lib_src}) - -# Do not output the shared library into the root -SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build) - -INSTALL (TARGETS charsetdetect DESTINATION lib) -INSTALL (FILES charsetdetect.h DESTINATION include) diff --git a/src/ext/libcharsetdetect/Makefile.test b/src/ext/libcharsetdetect/Makefile.test deleted file mode 100644 index 80cae1c..0000000 --- a/src/ext/libcharsetdetect/Makefile.test +++ /dev/null @@ -1,6 +0,0 @@ -trchardet: trchardet.o build/libcharsetdetect.a - $(CXX) -g -o trchardet trchardet.o build/libcharsetdetect.a - - -trchardet.o : trchardet.c charsetdetect.h - $(CC) -g $(CFLAGS) -c trchardet.c diff --git a/src/ext/libcharsetdetect/README.md b/src/ext/libcharsetdetect/README.md deleted file mode 100644 index 12e368e..0000000 --- a/src/ext/libcharsetdetect/README.md +++ /dev/null @@ -1,152 +0,0 @@ -# Universal Character Set Detector (UCSD) - -A library exposing a C interface and dependency-free interface to the Mozilla C++ UCSD library. - -This library provides a highly accurate set of heuristics that attempt to determine the character set used to encode some input text. -This is extremely useful when your program has to handle an input file which is supplied without any encoding metadata. - -Pulls together: - - * A NSPR emulation library (see `nspr-emu/README.md`) - * Code written by Colin Snover to provide a command line interface to the library - * The UCSD library itself from the Mozilla seamonkey source tree - -The UCSD version provided is that present in the Mozilla public repo as of 31/10/2010. - -## Building - -We have a build system based on CMake, so you will need that installed. That done, simply do this incantation: - - ./configure - make - sudo make install - -This will install the header file `charsetdetect.h` and the UCSD shared library, which you should link against in your compiler. - -## API documentation - -The library provides an opaque type of character set detectors: - - typedef void* csd_t; - -The first thing a client should do is create one of these: - - csd_t csd_open(void); - -A `csd_t` created in this fashion must be freed by `csd_close`. If creation fails, `csd_open` returns `(csd_t)-1`. - -Now you need to feed some data to the detector: - - int csd_consider(csd_t csd, const char *data, int length); - -The meaning of the return code is as follows: - - * Returns 0 if more data is needed to come to a conclusion - * Returns a positive number if enough data has been received to detect the character set - * Returns a negative number if there is an error - -Finally, close the detector to find out what the character set is: - - const char *csd_close(csd_t csd); - -The detected character set name is returned as an ASCII string. This function returns `NULL` if detection failed because there was not -enough data. It is safe to call `csd_close` at any point from creation by `csd_open` to the first call of `csd_close` on that character -set detector. - -## Full example - -This is a complete C program that shows how the library can be used to build a simple command-line character set detector: - - #include "charsetdetect.h" - #include "stdio.h" - - #define BUFFER_SIZE 4096 - - int main(int argc, const char * argv[]) { - csd_t csd = csd_open(); - if (csd == (csd_t)-1) { - printf("csd_open failed\n"); - return 1; - } - - int size; - char buf[BUFFER_SIZE] = {0}; - - while ((size = fread(buf, 1, sizeof(buf), stdin)) != 0) { - int result = csd_consider(csd, buf, size); - if (result < 0) { - printf("csd_consider failed\n"); - return 3; - } else if (result > 0) { - // Already have enough data - break; - } - } - - const char *result = csd_close(csd); - if (result == NULL) { - printf("Unknown character set\n"); - return 2; - } else { - printf("%s\n", result); - return 0; - } - } - -You can compile it and try it (on platforms with GCC) as follows: - - gcc example.c -lcharsetdetect - ./a.out < my_test_file.txt - -## Known character sets - -The list of possible character sets that can be returned from the library as of the most recent update are: - - Big5 - EUC-JP - EUC-KR - GB18030 - gb18030 - HZ-GB-2312 - IBM855 - IBM866 - ISO-2022-CN - ISO-2022-JP - ISO-2022-KR - ISO-8859-2 - ISO-8859-5 - ISO-8859-7 - ISO-8859-8 - KOI8-R - Shift_JIS - TIS-620 - UTF-8 - UTF-16BE - UTF-16LE - UTF-32BE - UTF-32LE - windows-1250 - windows-1251 - windows-1252 - windows-1253 - windows-1255 - x-euc-tw - X-ISO-10646-UCS-4-2143 - X-ISO-10646-UCS-4-3412 - x-mac-cyrillic - -We believe this list to be exhaustive. Future updates to the UCSD library may add more alternatives, but we will endeavour to keep -this list current. - -Notice that you may get both capitalisations of `GB18030`. For this reason (and to be future-proof against any future behaviour -like this for newly-added character sets) we recommend that you compare character set names case insensitively. - -## Licensing - -The files `libcharsetdetect.{cpp,h}` are (c) 2010 Colin Snover and released under an MIT license. - -The UCSD is (c) mozilla.org and tri-licensed under MPL 1.1/GPL 2.0/LGPL 2.1. - -We incorporate header files from the NSPR emulation library, which is LGPL licensed. - -Thus the resulting artifact is LGPL licensed (I think). \ No newline at end of file diff --git a/src/ext/libcharsetdetect/charsetdetect.cpp b/src/ext/libcharsetdetect/charsetdetect.cpp deleted file mode 100644 index 01c13a8..0000000 --- a/src/ext/libcharsetdetect/charsetdetect.cpp +++ /dev/null @@ -1,94 +0,0 @@ -#include -#include "charsetdetect.h" -#include "charsetdetectPriv.h" - - -// -// C++ API to the character set detector (not exported) -// - -void Detector::Report(const char* aCharset) { - // mDone has to be set true here because the original code - // does not always set it, and I am trying to avoid modifying - // the original code. - mDone = PR_TRUE; - - mDetectedCharset = aCharset; -} - -int Detector::Consider(const char *data, int length) { - if (HandleData(data, length) == NS_ERROR_OUT_OF_MEMORY) { - // Error, signal with a negative number - return -1; - } - - if (mDone) { - // Detected early - return 0; - } - - // Need more data! - return 1; -} - -const char *Detector::Close(void) { - DataEnd(); - - if (!mDone) { - if (mInputState == eEscAscii) { - return "ibm850"; - } - else if (mInputState == ePureAscii) { - return "ASCII"; - } - - return NULL; - } - - return mDetectedCharset; -} - -// -// C API to the character set detector (we actually export this) -// - -csd_t csd_open(void) { - // TODO: capture exceptions thrown by "new" and return -1 in that case - // TODO: provide C-land with access to the language filter constructor argument - return new Detector(NS_FILTER_ALL); -} - -int csd_consider(csd_t csd, const char *data, int length) { - return ((Detector*)csd)->Consider(data, length); -} - -const char *csd_close(csd_t csd) { - const char *result = ((Detector*)csd)->Close(); - delete ((Detector*)csd); - return result; -} - -///* -const char *Detector::Close2(float *confidence) { - DataEnd2(confidence); - - if (!mDone) { - if (mInputState == eEscAscii) { - return "ibm850"; - } - else if (mInputState == ePureAscii) { - return "ASCII"; - } - - return NULL; - } - - return mDetectedCharset; -} - -const char *csd_close2(csd_t csd,float *confidence) { - const char *result = ((Detector*)csd)->Close2(confidence); - delete ((Detector*)csd); - return result; -} -//*/ \ No newline at end of file diff --git a/src/ext/libcharsetdetect/charsetdetect.h b/src/ext/libcharsetdetect/charsetdetect.h deleted file mode 100644 index 6cfd1d0..0000000 --- a/src/ext/libcharsetdetect/charsetdetect.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef charsetdetect_ -#define charsetdetect_ - -/* The classes below are exported */ -#pragma GCC visibility push(default) - -#ifdef __cplusplus -extern "C" { -#endif - -// Opaque type of character set detectors -typedef void* csd_t; - -// Create a new character set detector. Must be freed by csd_close. -// If creation fails, returns (csd_t)-1. -csd_t csd_open(void); -// Feeds some more data to the character set detector. Returns 0 if it -// needs more data to come to a conclusion and a positive number if it has enough to say what -// the character set is. Returns a negative number if there is an error. -int csd_consider(csd_t csd, const char *data, int length); -// Closes the character set detector and returns the detected character set name as an ASCII string. -// Returns NULL if detection failed. -const char *csd_close(csd_t csd); - -const char *csd_close2(csd_t csd,float *confidence); - -#ifdef __cplusplus -} -#endif - -#pragma GCC visibility pop -#endif diff --git a/src/ext/libcharsetdetect/charsetdetectPriv.h b/src/ext/libcharsetdetect/charsetdetectPriv.h deleted file mode 100644 index 89c950b..0000000 --- a/src/ext/libcharsetdetect/charsetdetectPriv.h +++ /dev/null @@ -1,19 +0,0 @@ -/* The classes below are not exported */ - -#include -#include -#include "nscore.h" -#include "nsUniversalDetector.h" - -class Detector : public nsUniversalDetector { -public: - Detector(PRUint32 aLanguageFilter) : nsUniversalDetector(aLanguageFilter) {}; - int Consider(const char *data, int length); - const char *Close(void); - - const char *Close2(float *confidence); -protected: - void Report(const char* aCharset); - const char *mDetectedCharset; -}; - diff --git a/src/ext/libcharsetdetect/configure b/src/ext/libcharsetdetect/configure deleted file mode 100644 index 68b79c0..0000000 --- a/src/ext/libcharsetdetect/configure +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -cmake . -DBUILD_SHARED_LIBS=YES "$@" diff --git a/src/ext/libcharsetdetect/configure-iphone b/src/ext/libcharsetdetect/configure-iphone deleted file mode 100644 index 0b96fad..0000000 --- a/src/ext/libcharsetdetect/configure-iphone +++ /dev/null @@ -1,27 +0,0 @@ -#! /bin/bash -unset CPATH -unset C_INCLUDE_PATH -unset CPLUS_INCLUDE_PATH -unset OBJC_INCLUDE_PATH -unset LIBS -unset DYLD_FALLBACK_LIBRARY_PATH -unset DYLD_FALLBACK_FRAMEWORK_PATH - -if [ $# -lt 1 ]; then - echo "Usage: $0 sdk_version" - exit 1 -fi - -export SDKVER=$1 -shift - -export DEVROOT="/Developer/Platforms/iPhoneOS.platform/Developer" -export SDKROOT="$DEVROOT/SDKs/iPhoneOS$SDKVER.sdk" -export PKG_CONFIG_PATH="$SDKROOT/usr/lib/pkgconfig":"$SDKROOT/opt/iphone-$SDKVER/lib/pkgconfig":"$SDKROOT/usr/local/iphone-$SDKVER/lib/pkgconfig" -export PKG_CONFIG_LIBDIR="$PKG_CONFIG_PATH" - -# OK, this is really weird... running CMake can fail the first few times. It looks like you need to run it a few times to get -# the values that you SET to propagate around, because if you don't the linker flags etc that get used are outdated -for i in 1 2 3; do - cmake . -DSDKVER=$SDKVER -DCMAKE_TOOLCHAIN_FILE="cmake-toolchains/iphone.toolchain" -DCMAKE_INSTALL_PREFIX="/opt/iphone-$SDKVER" "$@" -done diff --git a/src/ext/libcharsetdetect/configure-iphone-simulator b/src/ext/libcharsetdetect/configure-iphone-simulator deleted file mode 100644 index 3de5886..0000000 --- a/src/ext/libcharsetdetect/configure-iphone-simulator +++ /dev/null @@ -1,27 +0,0 @@ -#! /bin/bash -unset CPATH -unset C_INCLUDE_PATH -unset CPLUS_INCLUDE_PATH -unset OBJC_INCLUDE_PATH -unset LIBS -unset DYLD_FALLBACK_LIBRARY_PATH -unset DYLD_FALLBACK_FRAMEWORK_PATH - -if [ $# -lt 1 ]; then - echo "Usage: $0 sdk_version" - exit 1 -fi - -export SDKVER=$1 -shift - -export DEVROOT="/Developer/Platforms/iPhoneSimulator.platform/Developer" -export SDKROOT="$DEVROOT/SDKs/iPhoneSimulator$SDKVER.sdk" -export PKG_CONFIG_PATH="$SDKROOT/usr/lib/pkgconfig":"$SDKROOT/opt/iphone-simulator-$SDKVER/lib/pkgconfig":"$SDKROOT/usr/local/iphone-simulator-$SDKVER/lib/pkgconfig" -export PKG_CONFIG_LIBDIR="$PKG_CONFIG_PATH" - -# OK, this is really weird... running CMake can fail the first few times. It looks like you need to run it a few times to get -# the values that you SET to propagate around, because if you don't the linker flags etc that get used are outdated -for i in 1 2 3; do - cmake . -DSDKVER=$SDKVER -DCMAKE_TOOLCHAIN_FILE="cmake-toolchains/iphone-simulator.toolchain" -DCMAKE_INSTALL_PREFIX="/opt/iphone-simulator-$SDKVER" "$@" -done diff --git a/src/ext/libcharsetdetect/main.cpp b/src/ext/libcharsetdetect/main.cpp deleted file mode 100644 index c9411ec..0000000 --- a/src/ext/libcharsetdetect/main.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "memory_leak.h" -#include "charsetdetect.h" -#include -#include -#include -using namespace std; - - -#define BUFFER_SIZE 100*1024 - -void main(){ - ::_CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); - csd_t csd = csd_open(); - if(csd == (csd_t)-1){ - std::cout << "csd_open faild\n"; - exit(1); - } - - //ifstream ifs("E:\\MySourcecode\\git\\cchardet\\test\\testdata\\bg\\ISO-8859-5\\wikitop_bg_ISO-8859-5.txt"); - //ifstream ifs("E:\\MySourcecode\\git\\cchardet\\test\\testdata\\bg\\UTF-8\\wikitop_bg_UTF-8.txt"); - ifstream ifs("E:\\MySourcecode\\git\\cchardet\\test\\testdata\\cz\\ISO-8859-2\\wikitop_cz_ISO-8859-2.txt"); - if(!ifs){ - std::cerr<<"Cant open the file\n"; - exit(1); - } - - int len; - ifs.seekg(0,std::ios::end); - len = ifs.tellg(); - ifs.seekg(0,std::ios::beg); - char* buf = new char[len]; - ifs.read(buf,len); - int result; - const char *result_; - std::cout<<"CLIENT SENDING More Data\n"; - result = csd_consider(csd, buf,len); - if(result < 0){ - std::cout<<"csd_consider failed\n"; - exit(1); - } - result_ = csd_close(csd); - if(result_ == NULL){ - std::cout<<"Unknown character set\n"; - }else{ - printf("%s\n", result_); - } - - delete buf; -} \ No newline at end of file diff --git a/src/ext/libcharsetdetect/memory_leak.h b/src/ext/libcharsetdetect/memory_leak.h deleted file mode 100644 index 05fd1c4..0000000 --- a/src/ext/libcharsetdetect/memory_leak.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef MEMORY_LEAK_H -#define MEMORY_LEAK_H -#define _CRTDBG_MAP_ALLOC -#include -#ifdef _DEBUG -#define new new(_NORMAL_BLOCK,__FILE__,__LINE__) -#endif -#endif \ No newline at end of file diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/Big5Freq.tab b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/Big5Freq.tab deleted file mode 100644 index fd6b0e0..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/Big5Freq.tab +++ /dev/null @@ -1,943 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -// Big5 frequency table -// by Taiwan's Mandarin Promotion Council -// - -/****************************************************************************** - * 128 --> 0.42261 - * 256 --> 0.57851 - * 512 --> 0.74851 - * 1024 --> 0.89384 - * 2048 --> 0.97583 - * - * Idea Distribution Ratio = 0.74851/(1-0.74851) =2.98 - * Random Distribution Ration = 512/(5401-512)=0.105 - * - * Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR - *****************************************************************************/ - -#define BIG5_TYPICAL_DISTRIBUTION_RATIO (float)0.75 - - -//Char to FreqOrder table , -#define BIG5_TABLE_SIZE 5376 - -static const PRInt16 Big5CharToFreqOrder[] = -{ - 1,1801,1506, 255,1431, 198, 9, 82, 6,5008, 177, 202,3681,1256,2821, 110, // 16 -3814, 33,3274, 261, 76, 44,2114, 16,2946,2187,1176, 659,3971, 26,3451,2653, // 32 -1198,3972,3350,4202, 410,2215, 302, 590, 361,1964, 8, 204, 58,4510,5009,1932, // 48 - 63,5010,5011, 317,1614, 75, 222, 159,4203,2417,1480,5012,3555,3091, 224,2822, // 64 -3682, 3, 10,3973,1471, 29,2787,1135,2866,1940, 873, 130,3275,1123, 312,5013, // 80 -4511,2052, 507, 252, 682,5014, 142,1915, 124, 206,2947, 34,3556,3204, 64, 604, // 96 -5015,2501,1977,1978, 155,1991, 645, 641,1606,5016,3452, 337, 72, 406,5017, 80, // 112 - 630, 238,3205,1509, 263, 939,1092,2654, 756,1440,1094,3453, 449, 69,2987, 591, // 128 - 179,2096, 471, 115,2035,1844, 60, 50,2988, 134, 806,1869, 734,2036,3454, 180, // 144 - 995,1607, 156, 537,2907, 688,5018, 319,1305, 779,2145, 514,2379, 298,4512, 359, // 160 -2502, 90,2716,1338, 663, 11, 906,1099,2553, 20,2441, 182, 532,1716,5019, 732, // 176 -1376,4204,1311,1420,3206, 25,2317,1056, 113, 399, 382,1950, 242,3455,2474, 529, // 192 -3276, 475,1447,3683,5020, 117, 21, 656, 810,1297,2300,2334,3557,5021, 126,4205, // 208 - 706, 456, 150, 613,4513, 71,1118,2037,4206, 145,3092, 85, 835, 486,2115,1246, // 224 -1426, 428, 727,1285,1015, 800, 106, 623, 303,1281,5022,2128,2359, 347,3815, 221, // 240 -3558,3135,5023,1956,1153,4207, 83, 296,1199,3093, 192, 624, 93,5024, 822,1898, // 256 -2823,3136, 795,2065, 991,1554,1542,1592, 27, 43,2867, 859, 139,1456, 860,4514, // 272 - 437, 712,3974, 164,2397,3137, 695, 211,3037,2097, 195,3975,1608,3559,3560,3684, // 288 -3976, 234, 811,2989,2098,3977,2233,1441,3561,1615,2380, 668,2077,1638, 305, 228, // 304 -1664,4515, 467, 415,5025, 262,2099,1593, 239, 108, 300, 200,1033, 512,1247,2078, // 320 -5026,5027,2176,3207,3685,2682, 593, 845,1062,3277, 88,1723,2038,3978,1951, 212, // 336 - 266, 152, 149, 468,1899,4208,4516, 77, 187,5028,3038, 37, 5,2990,5029,3979, // 352 -5030,5031, 39,2524,4517,2908,3208,2079, 55, 148, 74,4518, 545, 483,1474,1029, // 368 -1665, 217,1870,1531,3138,1104,2655,4209, 24, 172,3562, 900,3980,3563,3564,4519, // 384 - 32,1408,2824,1312, 329, 487,2360,2251,2717, 784,2683, 4,3039,3351,1427,1789, // 400 - 188, 109, 499,5032,3686,1717,1790, 888,1217,3040,4520,5033,3565,5034,3352,1520, // 416 -3687,3981, 196,1034, 775,5035,5036, 929,1816, 249, 439, 38,5037,1063,5038, 794, // 432 -3982,1435,2301, 46, 178,3278,2066,5039,2381,5040, 214,1709,4521, 804, 35, 707, // 448 - 324,3688,1601,2554, 140, 459,4210,5041,5042,1365, 839, 272, 978,2262,2580,3456, // 464 -2129,1363,3689,1423, 697, 100,3094, 48, 70,1231, 495,3139,2196,5043,1294,5044, // 480 -2080, 462, 586,1042,3279, 853, 256, 988, 185,2382,3457,1698, 434,1084,5045,3458, // 496 - 314,2625,2788,4522,2335,2336, 569,2285, 637,1817,2525, 757,1162,1879,1616,3459, // 512 - 287,1577,2116, 768,4523,1671,2868,3566,2526,1321,3816, 909,2418,5046,4211, 933, // 528 -3817,4212,2053,2361,1222,4524, 765,2419,1322, 786,4525,5047,1920,1462,1677,2909, // 544 -1699,5048,4526,1424,2442,3140,3690,2600,3353,1775,1941,3460,3983,4213, 309,1369, // 560 -1130,2825, 364,2234,1653,1299,3984,3567,3985,3986,2656, 525,1085,3041, 902,2001, // 576 -1475, 964,4527, 421,1845,1415,1057,2286, 940,1364,3141, 376,4528,4529,1381, 7, // 592 -2527, 983,2383, 336,1710,2684,1846, 321,3461, 559,1131,3042,2752,1809,1132,1313, // 608 - 265,1481,1858,5049, 352,1203,2826,3280, 167,1089, 420,2827, 776, 792,1724,3568, // 624 -4214,2443,3281,5050,4215,5051, 446, 229, 333,2753, 901,3818,1200,1557,4530,2657, // 640 -1921, 395,2754,2685,3819,4216,1836, 125, 916,3209,2626,4531,5052,5053,3820,5054, // 656 -5055,5056,4532,3142,3691,1133,2555,1757,3462,1510,2318,1409,3569,5057,2146, 438, // 672 -2601,2910,2384,3354,1068, 958,3043, 461, 311,2869,2686,4217,1916,3210,4218,1979, // 688 - 383, 750,2755,2627,4219, 274, 539, 385,1278,1442,5058,1154,1965, 384, 561, 210, // 704 - 98,1295,2556,3570,5059,1711,2420,1482,3463,3987,2911,1257, 129,5060,3821, 642, // 720 - 523,2789,2790,2658,5061, 141,2235,1333, 68, 176, 441, 876, 907,4220, 603,2602, // 736 - 710, 171,3464, 404, 549, 18,3143,2398,1410,3692,1666,5062,3571,4533,2912,4534, // 752 -5063,2991, 368,5064, 146, 366, 99, 871,3693,1543, 748, 807,1586,1185, 22,2263, // 768 - 379,3822,3211,5065,3212, 505,1942,2628,1992,1382,2319,5066, 380,2362, 218, 702, // 784 -1818,1248,3465,3044,3572,3355,3282,5067,2992,3694, 930,3283,3823,5068, 59,5069, // 800 - 585, 601,4221, 497,3466,1112,1314,4535,1802,5070,1223,1472,2177,5071, 749,1837, // 816 - 690,1900,3824,1773,3988,1476, 429,1043,1791,2236,2117, 917,4222, 447,1086,1629, // 832 -5072, 556,5073,5074,2021,1654, 844,1090, 105, 550, 966,1758,2828,1008,1783, 686, // 848 -1095,5075,2287, 793,1602,5076,3573,2603,4536,4223,2948,2302,4537,3825, 980,2503, // 864 - 544, 353, 527,4538, 908,2687,2913,5077, 381,2629,1943,1348,5078,1341,1252, 560, // 880 -3095,5079,3467,2870,5080,2054, 973, 886,2081, 143,4539,5081,5082, 157,3989, 496, // 896 -4224, 57, 840, 540,2039,4540,4541,3468,2118,1445, 970,2264,1748,1966,2082,4225, // 912 -3144,1234,1776,3284,2829,3695, 773,1206,2130,1066,2040,1326,3990,1738,1725,4226, // 928 - 279,3145, 51,1544,2604, 423,1578,2131,2067, 173,4542,1880,5083,5084,1583, 264, // 944 - 610,3696,4543,2444, 280, 154,5085,5086,5087,1739, 338,1282,3096, 693,2871,1411, // 960 -1074,3826,2445,5088,4544,5089,5090,1240, 952,2399,5091,2914,1538,2688, 685,1483, // 976 -4227,2475,1436, 953,4228,2055,4545, 671,2400, 79,4229,2446,3285, 608, 567,2689, // 992 -3469,4230,4231,1691, 393,1261,1792,2401,5092,4546,5093,5094,5095,5096,1383,1672, // 1008 -3827,3213,1464, 522,1119, 661,1150, 216, 675,4547,3991,1432,3574, 609,4548,2690, // 1024 -2402,5097,5098,5099,4232,3045, 0,5100,2476, 315, 231,2447, 301,3356,4549,2385, // 1040 -5101, 233,4233,3697,1819,4550,4551,5102, 96,1777,1315,2083,5103, 257,5104,1810, // 1056 -3698,2718,1139,1820,4234,2022,1124,2164,2791,1778,2659,5105,3097, 363,1655,3214, // 1072 -5106,2993,5107,5108,5109,3992,1567,3993, 718, 103,3215, 849,1443, 341,3357,2949, // 1088 -1484,5110,1712, 127, 67, 339,4235,2403, 679,1412, 821,5111,5112, 834, 738, 351, // 1104 -2994,2147, 846, 235,1497,1881, 418,1993,3828,2719, 186,1100,2148,2756,3575,1545, // 1120 -1355,2950,2872,1377, 583,3994,4236,2581,2995,5113,1298,3699,1078,2557,3700,2363, // 1136 - 78,3829,3830, 267,1289,2100,2002,1594,4237, 348, 369,1274,2197,2178,1838,4552, // 1152 -1821,2830,3701,2757,2288,2003,4553,2951,2758, 144,3358, 882,4554,3995,2759,3470, // 1168 -4555,2915,5114,4238,1726, 320,5115,3996,3046, 788,2996,5116,2831,1774,1327,2873, // 1184 -3997,2832,5117,1306,4556,2004,1700,3831,3576,2364,2660, 787,2023, 506, 824,3702, // 1200 - 534, 323,4557,1044,3359,2024,1901, 946,3471,5118,1779,1500,1678,5119,1882,4558, // 1216 - 165, 243,4559,3703,2528, 123, 683,4239, 764,4560, 36,3998,1793, 589,2916, 816, // 1232 - 626,1667,3047,2237,1639,1555,1622,3832,3999,5120,4000,2874,1370,1228,1933, 891, // 1248 -2084,2917, 304,4240,5121, 292,2997,2720,3577, 691,2101,4241,1115,4561, 118, 662, // 1264 -5122, 611,1156, 854,2386,1316,2875, 2, 386, 515,2918,5123,5124,3286, 868,2238, // 1280 -1486, 855,2661, 785,2216,3048,5125,1040,3216,3578,5126,3146, 448,5127,1525,5128, // 1296 -2165,4562,5129,3833,5130,4242,2833,3579,3147, 503, 818,4001,3148,1568, 814, 676, // 1312 -1444, 306,1749,5131,3834,1416,1030, 197,1428, 805,2834,1501,4563,5132,5133,5134, // 1328 -1994,5135,4564,5136,5137,2198, 13,2792,3704,2998,3149,1229,1917,5138,3835,2132, // 1344 -5139,4243,4565,2404,3580,5140,2217,1511,1727,1120,5141,5142, 646,3836,2448, 307, // 1360 -5143,5144,1595,3217,5145,5146,5147,3705,1113,1356,4002,1465,2529,2530,5148, 519, // 1376 -5149, 128,2133, 92,2289,1980,5150,4003,1512, 342,3150,2199,5151,2793,2218,1981, // 1392 -3360,4244, 290,1656,1317, 789, 827,2365,5152,3837,4566, 562, 581,4004,5153, 401, // 1408 -4567,2252, 94,4568,5154,1399,2794,5155,1463,2025,4569,3218,1944,5156, 828,1105, // 1424 -4245,1262,1394,5157,4246, 605,4570,5158,1784,2876,5159,2835, 819,2102, 578,2200, // 1440 -2952,5160,1502, 436,3287,4247,3288,2836,4005,2919,3472,3473,5161,2721,2320,5162, // 1456 -5163,2337,2068, 23,4571, 193, 826,3838,2103, 699,1630,4248,3098, 390,1794,1064, // 1472 -3581,5164,1579,3099,3100,1400,5165,4249,1839,1640,2877,5166,4572,4573, 137,4250, // 1488 - 598,3101,1967, 780, 104, 974,2953,5167, 278, 899, 253, 402, 572, 504, 493,1339, // 1504 -5168,4006,1275,4574,2582,2558,5169,3706,3049,3102,2253, 565,1334,2722, 863, 41, // 1520 -5170,5171,4575,5172,1657,2338, 19, 463,2760,4251, 606,5173,2999,3289,1087,2085, // 1536 -1323,2662,3000,5174,1631,1623,1750,4252,2691,5175,2878, 791,2723,2663,2339, 232, // 1552 -2421,5176,3001,1498,5177,2664,2630, 755,1366,3707,3290,3151,2026,1609, 119,1918, // 1568 -3474, 862,1026,4253,5178,4007,3839,4576,4008,4577,2265,1952,2477,5179,1125, 817, // 1584 -4254,4255,4009,1513,1766,2041,1487,4256,3050,3291,2837,3840,3152,5180,5181,1507, // 1600 -5182,2692, 733, 40,1632,1106,2879, 345,4257, 841,2531, 230,4578,3002,1847,3292, // 1616 -3475,5183,1263, 986,3476,5184, 735, 879, 254,1137, 857, 622,1300,1180,1388,1562, // 1632 -4010,4011,2954, 967,2761,2665,1349, 592,2134,1692,3361,3003,1995,4258,1679,4012, // 1648 -1902,2188,5185, 739,3708,2724,1296,1290,5186,4259,2201,2202,1922,1563,2605,2559, // 1664 -1871,2762,3004,5187, 435,5188, 343,1108, 596, 17,1751,4579,2239,3477,3709,5189, // 1680 -4580, 294,3582,2955,1693, 477, 979, 281,2042,3583, 643,2043,3710,2631,2795,2266, // 1696 -1031,2340,2135,2303,3584,4581, 367,1249,2560,5190,3585,5191,4582,1283,3362,2005, // 1712 - 240,1762,3363,4583,4584, 836,1069,3153, 474,5192,2149,2532, 268,3586,5193,3219, // 1728 -1521,1284,5194,1658,1546,4260,5195,3587,3588,5196,4261,3364,2693,1685,4262, 961, // 1744 -1673,2632, 190,2006,2203,3841,4585,4586,5197, 570,2504,3711,1490,5198,4587,2633, // 1760 -3293,1957,4588, 584,1514, 396,1045,1945,5199,4589,1968,2449,5200,5201,4590,4013, // 1776 - 619,5202,3154,3294, 215,2007,2796,2561,3220,4591,3221,4592, 763,4263,3842,4593, // 1792 -5203,5204,1958,1767,2956,3365,3712,1174, 452,1477,4594,3366,3155,5205,2838,1253, // 1808 -2387,2189,1091,2290,4264, 492,5206, 638,1169,1825,2136,1752,4014, 648, 926,1021, // 1824 -1324,4595, 520,4596, 997, 847,1007, 892,4597,3843,2267,1872,3713,2405,1785,4598, // 1840 -1953,2957,3103,3222,1728,4265,2044,3714,4599,2008,1701,3156,1551, 30,2268,4266, // 1856 -5207,2027,4600,3589,5208, 501,5209,4267, 594,3478,2166,1822,3590,3479,3591,3223, // 1872 - 829,2839,4268,5210,1680,3157,1225,4269,5211,3295,4601,4270,3158,2341,5212,4602, // 1888 -4271,5213,4015,4016,5214,1848,2388,2606,3367,5215,4603, 374,4017, 652,4272,4273, // 1904 - 375,1140, 798,5216,5217,5218,2366,4604,2269, 546,1659, 138,3051,2450,4605,5219, // 1920 -2254, 612,1849, 910, 796,3844,1740,1371, 825,3845,3846,5220,2920,2562,5221, 692, // 1936 - 444,3052,2634, 801,4606,4274,5222,1491, 244,1053,3053,4275,4276, 340,5223,4018, // 1952 -1041,3005, 293,1168, 87,1357,5224,1539, 959,5225,2240, 721, 694,4277,3847, 219, // 1968 -1478, 644,1417,3368,2666,1413,1401,1335,1389,4019,5226,5227,3006,2367,3159,1826, // 1984 - 730,1515, 184,2840, 66,4607,5228,1660,2958, 246,3369, 378,1457, 226,3480, 975, // 2000 -4020,2959,1264,3592, 674, 696,5229, 163,5230,1141,2422,2167, 713,3593,3370,4608, // 2016 -4021,5231,5232,1186, 15,5233,1079,1070,5234,1522,3224,3594, 276,1050,2725, 758, // 2032 -1126, 653,2960,3296,5235,2342, 889,3595,4022,3104,3007, 903,1250,4609,4023,3481, // 2048 -3596,1342,1681,1718, 766,3297, 286, 89,2961,3715,5236,1713,5237,2607,3371,3008, // 2064 -5238,2962,2219,3225,2880,5239,4610,2505,2533, 181, 387,1075,4024, 731,2190,3372, // 2080 -5240,3298, 310, 313,3482,2304, 770,4278, 54,3054, 189,4611,3105,3848,4025,5241, // 2096 -1230,1617,1850, 355,3597,4279,4612,3373, 111,4280,3716,1350,3160,3483,3055,4281, // 2112 -2150,3299,3598,5242,2797,4026,4027,3009, 722,2009,5243,1071, 247,1207,2343,2478, // 2128 -1378,4613,2010, 864,1437,1214,4614, 373,3849,1142,2220, 667,4615, 442,2763,2563, // 2144 -3850,4028,1969,4282,3300,1840, 837, 170,1107, 934,1336,1883,5244,5245,2119,4283, // 2160 -2841, 743,1569,5246,4616,4284, 582,2389,1418,3484,5247,1803,5248, 357,1395,1729, // 2176 -3717,3301,2423,1564,2241,5249,3106,3851,1633,4617,1114,2086,4285,1532,5250, 482, // 2192 -2451,4618,5251,5252,1492, 833,1466,5253,2726,3599,1641,2842,5254,1526,1272,3718, // 2208 -4286,1686,1795, 416,2564,1903,1954,1804,5255,3852,2798,3853,1159,2321,5256,2881, // 2224 -4619,1610,1584,3056,2424,2764, 443,3302,1163,3161,5257,5258,4029,5259,4287,2506, // 2240 -3057,4620,4030,3162,2104,1647,3600,2011,1873,4288,5260,4289, 431,3485,5261, 250, // 2256 - 97, 81,4290,5262,1648,1851,1558, 160, 848,5263, 866, 740,1694,5264,2204,2843, // 2272 -3226,4291,4621,3719,1687, 950,2479, 426, 469,3227,3720,3721,4031,5265,5266,1188, // 2288 - 424,1996, 861,3601,4292,3854,2205,2694, 168,1235,3602,4293,5267,2087,1674,4622, // 2304 -3374,3303, 220,2565,1009,5268,3855, 670,3010, 332,1208, 717,5269,5270,3603,2452, // 2320 -4032,3375,5271, 513,5272,1209,2882,3376,3163,4623,1080,5273,5274,5275,5276,2534, // 2336 -3722,3604, 815,1587,4033,4034,5277,3605,3486,3856,1254,4624,1328,3058,1390,4035, // 2352 -1741,4036,3857,4037,5278, 236,3858,2453,3304,5279,5280,3723,3859,1273,3860,4625, // 2368 -5281, 308,5282,4626, 245,4627,1852,2480,1307,2583, 430, 715,2137,2454,5283, 270, // 2384 - 199,2883,4038,5284,3606,2727,1753, 761,1754, 725,1661,1841,4628,3487,3724,5285, // 2400 -5286, 587, 14,3305, 227,2608, 326, 480,2270, 943,2765,3607, 291, 650,1884,5287, // 2416 -1702,1226, 102,1547, 62,3488, 904,4629,3489,1164,4294,5288,5289,1224,1548,2766, // 2432 - 391, 498,1493,5290,1386,1419,5291,2056,1177,4630, 813, 880,1081,2368, 566,1145, // 2448 -4631,2291,1001,1035,2566,2609,2242, 394,1286,5292,5293,2069,5294, 86,1494,1730, // 2464 -4039, 491,1588, 745, 897,2963, 843,3377,4040,2767,2884,3306,1768, 998,2221,2070, // 2480 - 397,1827,1195,1970,3725,3011,3378, 284,5295,3861,2507,2138,2120,1904,5296,4041, // 2496 -2151,4042,4295,1036,3490,1905, 114,2567,4296, 209,1527,5297,5298,2964,2844,2635, // 2512 -2390,2728,3164, 812,2568,5299,3307,5300,1559, 737,1885,3726,1210, 885, 28,2695, // 2528 -3608,3862,5301,4297,1004,1780,4632,5302, 346,1982,2222,2696,4633,3863,1742, 797, // 2544 -1642,4043,1934,1072,1384,2152, 896,4044,3308,3727,3228,2885,3609,5303,2569,1959, // 2560 -4634,2455,1786,5304,5305,5306,4045,4298,1005,1308,3728,4299,2729,4635,4636,1528, // 2576 -2610, 161,1178,4300,1983, 987,4637,1101,4301, 631,4046,1157,3229,2425,1343,1241, // 2592 -1016,2243,2570, 372, 877,2344,2508,1160, 555,1935, 911,4047,5307, 466,1170, 169, // 2608 -1051,2921,2697,3729,2481,3012,1182,2012,2571,1251,2636,5308, 992,2345,3491,1540, // 2624 -2730,1201,2071,2406,1997,2482,5309,4638, 528,1923,2191,1503,1874,1570,2369,3379, // 2640 -3309,5310, 557,1073,5311,1828,3492,2088,2271,3165,3059,3107, 767,3108,2799,4639, // 2656 -1006,4302,4640,2346,1267,2179,3730,3230, 778,4048,3231,2731,1597,2667,5312,4641, // 2672 -5313,3493,5314,5315,5316,3310,2698,1433,3311, 131, 95,1504,4049, 723,4303,3166, // 2688 -1842,3610,2768,2192,4050,2028,2105,3731,5317,3013,4051,1218,5318,3380,3232,4052, // 2704 -4304,2584, 248,1634,3864, 912,5319,2845,3732,3060,3865, 654, 53,5320,3014,5321, // 2720 -1688,4642, 777,3494,1032,4053,1425,5322, 191, 820,2121,2846, 971,4643, 931,3233, // 2736 - 135, 664, 783,3866,1998, 772,2922,1936,4054,3867,4644,2923,3234, 282,2732, 640, // 2752 -1372,3495,1127, 922, 325,3381,5323,5324, 711,2045,5325,5326,4055,2223,2800,1937, // 2768 -4056,3382,2224,2255,3868,2305,5327,4645,3869,1258,3312,4057,3235,2139,2965,4058, // 2784 -4059,5328,2225, 258,3236,4646, 101,1227,5329,3313,1755,5330,1391,3314,5331,2924, // 2800 -2057, 893,5332,5333,5334,1402,4305,2347,5335,5336,3237,3611,5337,5338, 878,1325, // 2816 -1781,2801,4647, 259,1385,2585, 744,1183,2272,4648,5339,4060,2509,5340, 684,1024, // 2832 -4306,5341, 472,3612,3496,1165,3315,4061,4062, 322,2153, 881, 455,1695,1152,1340, // 2848 - 660, 554,2154,4649,1058,4650,4307, 830,1065,3383,4063,4651,1924,5342,1703,1919, // 2864 -5343, 932,2273, 122,5344,4652, 947, 677,5345,3870,2637, 297,1906,1925,2274,4653, // 2880 -2322,3316,5346,5347,4308,5348,4309, 84,4310, 112, 989,5349, 547,1059,4064, 701, // 2896 -3613,1019,5350,4311,5351,3497, 942, 639, 457,2306,2456, 993,2966, 407, 851, 494, // 2912 -4654,3384, 927,5352,1237,5353,2426,3385, 573,4312, 680, 921,2925,1279,1875, 285, // 2928 - 790,1448,1984, 719,2168,5354,5355,4655,4065,4066,1649,5356,1541, 563,5357,1077, // 2944 -5358,3386,3061,3498, 511,3015,4067,4068,3733,4069,1268,2572,3387,3238,4656,4657, // 2960 -5359, 535,1048,1276,1189,2926,2029,3167,1438,1373,2847,2967,1134,2013,5360,4313, // 2976 -1238,2586,3109,1259,5361, 700,5362,2968,3168,3734,4314,5363,4315,1146,1876,1907, // 2992 -4658,2611,4070, 781,2427, 132,1589, 203, 147, 273,2802,2407, 898,1787,2155,4071, // 3008 -4072,5364,3871,2803,5365,5366,4659,4660,5367,3239,5368,1635,3872, 965,5369,1805, // 3024 -2699,1516,3614,1121,1082,1329,3317,4073,1449,3873, 65,1128,2848,2927,2769,1590, // 3040 -3874,5370,5371, 12,2668, 45, 976,2587,3169,4661, 517,2535,1013,1037,3240,5372, // 3056 -3875,2849,5373,3876,5374,3499,5375,2612, 614,1999,2323,3877,3110,2733,2638,5376, // 3072 -2588,4316, 599,1269,5377,1811,3735,5378,2700,3111, 759,1060, 489,1806,3388,3318, // 3088 -1358,5379,5380,2391,1387,1215,2639,2256, 490,5381,5382,4317,1759,2392,2348,5383, // 3104 -4662,3878,1908,4074,2640,1807,3241,4663,3500,3319,2770,2349, 874,5384,5385,3501, // 3120 -3736,1859, 91,2928,3737,3062,3879,4664,5386,3170,4075,2669,5387,3502,1202,1403, // 3136 -3880,2969,2536,1517,2510,4665,3503,2511,5388,4666,5389,2701,1886,1495,1731,4076, // 3152 -2370,4667,5390,2030,5391,5392,4077,2702,1216, 237,2589,4318,2324,4078,3881,4668, // 3168 -4669,2703,3615,3504, 445,4670,5393,5394,5395,5396,2771, 61,4079,3738,1823,4080, // 3184 -5397, 687,2046, 935, 925, 405,2670, 703,1096,1860,2734,4671,4081,1877,1367,2704, // 3200 -3389, 918,2106,1782,2483, 334,3320,1611,1093,4672, 564,3171,3505,3739,3390, 945, // 3216 -2641,2058,4673,5398,1926, 872,4319,5399,3506,2705,3112, 349,4320,3740,4082,4674, // 3232 -3882,4321,3741,2156,4083,4675,4676,4322,4677,2408,2047, 782,4084, 400, 251,4323, // 3248 -1624,5400,5401, 277,3742, 299,1265, 476,1191,3883,2122,4324,4325,1109, 205,5402, // 3264 -2590,1000,2157,3616,1861,5403,5404,5405,4678,5406,4679,2573, 107,2484,2158,4085, // 3280 -3507,3172,5407,1533, 541,1301, 158, 753,4326,2886,3617,5408,1696, 370,1088,4327, // 3296 -4680,3618, 579, 327, 440, 162,2244, 269,1938,1374,3508, 968,3063, 56,1396,3113, // 3312 -2107,3321,3391,5409,1927,2159,4681,3016,5410,3619,5411,5412,3743,4682,2485,5413, // 3328 -2804,5414,1650,4683,5415,2613,5416,5417,4086,2671,3392,1149,3393,4087,3884,4088, // 3344 -5418,1076, 49,5419, 951,3242,3322,3323, 450,2850, 920,5420,1812,2805,2371,4328, // 3360 -1909,1138,2372,3885,3509,5421,3243,4684,1910,1147,1518,2428,4685,3886,5422,4686, // 3376 -2393,2614, 260,1796,3244,5423,5424,3887,3324, 708,5425,3620,1704,5426,3621,1351, // 3392 -1618,3394,3017,1887, 944,4329,3395,4330,3064,3396,4331,5427,3744, 422, 413,1714, // 3408 -3325, 500,2059,2350,4332,2486,5428,1344,1911, 954,5429,1668,5430,5431,4089,2409, // 3424 -4333,3622,3888,4334,5432,2307,1318,2512,3114, 133,3115,2887,4687, 629, 31,2851, // 3440 -2706,3889,4688, 850, 949,4689,4090,2970,1732,2089,4335,1496,1853,5433,4091, 620, // 3456 -3245, 981,1242,3745,3397,1619,3746,1643,3326,2140,2457,1971,1719,3510,2169,5434, // 3472 -3246,5435,5436,3398,1829,5437,1277,4690,1565,2048,5438,1636,3623,3116,5439, 869, // 3488 -2852, 655,3890,3891,3117,4092,3018,3892,1310,3624,4691,5440,5441,5442,1733, 558, // 3504 -4692,3747, 335,1549,3065,1756,4336,3748,1946,3511,1830,1291,1192, 470,2735,2108, // 3520 -2806, 913,1054,4093,5443,1027,5444,3066,4094,4693, 982,2672,3399,3173,3512,3247, // 3536 -3248,1947,2807,5445, 571,4694,5446,1831,5447,3625,2591,1523,2429,5448,2090, 984, // 3552 -4695,3749,1960,5449,3750, 852, 923,2808,3513,3751, 969,1519, 999,2049,2325,1705, // 3568 -5450,3118, 615,1662, 151, 597,4095,2410,2326,1049, 275,4696,3752,4337, 568,3753, // 3584 -3626,2487,4338,3754,5451,2430,2275, 409,3249,5452,1566,2888,3514,1002, 769,2853, // 3600 - 194,2091,3174,3755,2226,3327,4339, 628,1505,5453,5454,1763,2180,3019,4096, 521, // 3616 -1161,2592,1788,2206,2411,4697,4097,1625,4340,4341, 412, 42,3119, 464,5455,2642, // 3632 -4698,3400,1760,1571,2889,3515,2537,1219,2207,3893,2643,2141,2373,4699,4700,3328, // 3648 -1651,3401,3627,5456,5457,3628,2488,3516,5458,3756,5459,5460,2276,2092, 460,5461, // 3664 -4701,5462,3020, 962, 588,3629, 289,3250,2644,1116, 52,5463,3067,1797,5464,5465, // 3680 -5466,1467,5467,1598,1143,3757,4342,1985,1734,1067,4702,1280,3402, 465,4703,1572, // 3696 - 510,5468,1928,2245,1813,1644,3630,5469,4704,3758,5470,5471,2673,1573,1534,5472, // 3712 -5473, 536,1808,1761,3517,3894,3175,2645,5474,5475,5476,4705,3518,2929,1912,2809, // 3728 -5477,3329,1122, 377,3251,5478, 360,5479,5480,4343,1529, 551,5481,2060,3759,1769, // 3744 -2431,5482,2930,4344,3330,3120,2327,2109,2031,4706,1404, 136,1468,1479, 672,1171, // 3760 -3252,2308, 271,3176,5483,2772,5484,2050, 678,2736, 865,1948,4707,5485,2014,4098, // 3776 -2971,5486,2737,2227,1397,3068,3760,4708,4709,1735,2931,3403,3631,5487,3895, 509, // 3792 -2854,2458,2890,3896,5488,5489,3177,3178,4710,4345,2538,4711,2309,1166,1010, 552, // 3808 - 681,1888,5490,5491,2972,2973,4099,1287,1596,1862,3179, 358, 453, 736, 175, 478, // 3824 -1117, 905,1167,1097,5492,1854,1530,5493,1706,5494,2181,3519,2292,3761,3520,3632, // 3840 -4346,2093,4347,5495,3404,1193,2489,4348,1458,2193,2208,1863,1889,1421,3331,2932, // 3856 -3069,2182,3521, 595,2123,5496,4100,5497,5498,4349,1707,2646, 223,3762,1359, 751, // 3872 -3121, 183,3522,5499,2810,3021, 419,2374, 633, 704,3897,2394, 241,5500,5501,5502, // 3888 - 838,3022,3763,2277,2773,2459,3898,1939,2051,4101,1309,3122,2246,1181,5503,1136, // 3904 -2209,3899,2375,1446,4350,2310,4712,5504,5505,4351,1055,2615, 484,3764,5506,4102, // 3920 - 625,4352,2278,3405,1499,4353,4103,5507,4104,4354,3253,2279,2280,3523,5508,5509, // 3936 -2774, 808,2616,3765,3406,4105,4355,3123,2539, 526,3407,3900,4356, 955,5510,1620, // 3952 -4357,2647,2432,5511,1429,3766,1669,1832, 994, 928,5512,3633,1260,5513,5514,5515, // 3968 -1949,2293, 741,2933,1626,4358,2738,2460, 867,1184, 362,3408,1392,5516,5517,4106, // 3984 -4359,1770,1736,3254,2934,4713,4714,1929,2707,1459,1158,5518,3070,3409,2891,1292, // 4000 -1930,2513,2855,3767,1986,1187,2072,2015,2617,4360,5519,2574,2514,2170,3768,2490, // 4016 -3332,5520,3769,4715,5521,5522, 666,1003,3023,1022,3634,4361,5523,4716,1814,2257, // 4032 - 574,3901,1603, 295,1535, 705,3902,4362, 283, 858, 417,5524,5525,3255,4717,4718, // 4048 -3071,1220,1890,1046,2281,2461,4107,1393,1599, 689,2575, 388,4363,5526,2491, 802, // 4064 -5527,2811,3903,2061,1405,2258,5528,4719,3904,2110,1052,1345,3256,1585,5529, 809, // 4080 -5530,5531,5532, 575,2739,3524, 956,1552,1469,1144,2328,5533,2329,1560,2462,3635, // 4096 -3257,4108, 616,2210,4364,3180,2183,2294,5534,1833,5535,3525,4720,5536,1319,3770, // 4112 -3771,1211,3636,1023,3258,1293,2812,5537,5538,5539,3905, 607,2311,3906, 762,2892, // 4128 -1439,4365,1360,4721,1485,3072,5540,4722,1038,4366,1450,2062,2648,4367,1379,4723, // 4144 -2593,5541,5542,4368,1352,1414,2330,2935,1172,5543,5544,3907,3908,4724,1798,1451, // 4160 -5545,5546,5547,5548,2936,4109,4110,2492,2351, 411,4111,4112,3637,3333,3124,4725, // 4176 -1561,2674,1452,4113,1375,5549,5550, 47,2974, 316,5551,1406,1591,2937,3181,5552, // 4192 -1025,2142,3125,3182, 354,2740, 884,2228,4369,2412, 508,3772, 726,3638, 996,2433, // 4208 -3639, 729,5553, 392,2194,1453,4114,4726,3773,5554,5555,2463,3640,2618,1675,2813, // 4224 - 919,2352,2975,2353,1270,4727,4115, 73,5556,5557, 647,5558,3259,2856,2259,1550, // 4240 -1346,3024,5559,1332, 883,3526,5560,5561,5562,5563,3334,2775,5564,1212, 831,1347, // 4256 -4370,4728,2331,3909,1864,3073, 720,3910,4729,4730,3911,5565,4371,5566,5567,4731, // 4272 -5568,5569,1799,4732,3774,2619,4733,3641,1645,2376,4734,5570,2938, 669,2211,2675, // 4288 -2434,5571,2893,5572,5573,1028,3260,5574,4372,2413,5575,2260,1353,5576,5577,4735, // 4304 -3183, 518,5578,4116,5579,4373,1961,5580,2143,4374,5581,5582,3025,2354,2355,3912, // 4320 - 516,1834,1454,4117,2708,4375,4736,2229,2620,1972,1129,3642,5583,2776,5584,2976, // 4336 -1422, 577,1470,3026,1524,3410,5585,5586, 432,4376,3074,3527,5587,2594,1455,2515, // 4352 -2230,1973,1175,5588,1020,2741,4118,3528,4737,5589,2742,5590,1743,1361,3075,3529, // 4368 -2649,4119,4377,4738,2295, 895, 924,4378,2171, 331,2247,3076, 166,1627,3077,1098, // 4384 -5591,1232,2894,2231,3411,4739, 657, 403,1196,2377, 542,3775,3412,1600,4379,3530, // 4400 -5592,4740,2777,3261, 576, 530,1362,4741,4742,2540,2676,3776,4120,5593, 842,3913, // 4416 -5594,2814,2032,1014,4121, 213,2709,3413, 665, 621,4380,5595,3777,2939,2435,5596, // 4432 -2436,3335,3643,3414,4743,4381,2541,4382,4744,3644,1682,4383,3531,1380,5597, 724, // 4448 -2282, 600,1670,5598,1337,1233,4745,3126,2248,5599,1621,4746,5600, 651,4384,5601, // 4464 -1612,4385,2621,5602,2857,5603,2743,2312,3078,5604, 716,2464,3079, 174,1255,2710, // 4480 -4122,3645, 548,1320,1398, 728,4123,1574,5605,1891,1197,3080,4124,5606,3081,3082, // 4496 -3778,3646,3779, 747,5607, 635,4386,4747,5608,5609,5610,4387,5611,5612,4748,5613, // 4512 -3415,4749,2437, 451,5614,3780,2542,2073,4388,2744,4389,4125,5615,1764,4750,5616, // 4528 -4390, 350,4751,2283,2395,2493,5617,4391,4126,2249,1434,4127, 488,4752, 458,4392, // 4544 -4128,3781, 771,1330,2396,3914,2576,3184,2160,2414,1553,2677,3185,4393,5618,2494, // 4560 -2895,2622,1720,2711,4394,3416,4753,5619,2543,4395,5620,3262,4396,2778,5621,2016, // 4576 -2745,5622,1155,1017,3782,3915,5623,3336,2313, 201,1865,4397,1430,5624,4129,5625, // 4592 -5626,5627,5628,5629,4398,1604,5630, 414,1866, 371,2595,4754,4755,3532,2017,3127, // 4608 -4756,1708, 960,4399, 887, 389,2172,1536,1663,1721,5631,2232,4130,2356,2940,1580, // 4624 -5632,5633,1744,4757,2544,4758,4759,5634,4760,5635,2074,5636,4761,3647,3417,2896, // 4640 -4400,5637,4401,2650,3418,2815, 673,2712,2465, 709,3533,4131,3648,4402,5638,1148, // 4656 - 502, 634,5639,5640,1204,4762,3649,1575,4763,2623,3783,5641,3784,3128, 948,3263, // 4672 - 121,1745,3916,1110,5642,4403,3083,2516,3027,4132,3785,1151,1771,3917,1488,4133, // 4688 -1987,5643,2438,3534,5644,5645,2094,5646,4404,3918,1213,1407,2816, 531,2746,2545, // 4704 -3264,1011,1537,4764,2779,4405,3129,1061,5647,3786,3787,1867,2897,5648,2018, 120, // 4720 -4406,4407,2063,3650,3265,2314,3919,2678,3419,1955,4765,4134,5649,3535,1047,2713, // 4736 -1266,5650,1368,4766,2858, 649,3420,3920,2546,2747,1102,2859,2679,5651,5652,2000, // 4752 -5653,1111,3651,2977,5654,2495,3921,3652,2817,1855,3421,3788,5655,5656,3422,2415, // 4768 -2898,3337,3266,3653,5657,2577,5658,3654,2818,4135,1460, 856,5659,3655,5660,2899, // 4784 -2978,5661,2900,3922,5662,4408, 632,2517, 875,3923,1697,3924,2296,5663,5664,4767, // 4800 -3028,1239, 580,4768,4409,5665, 914, 936,2075,1190,4136,1039,2124,5666,5667,5668, // 4816 -5669,3423,1473,5670,1354,4410,3925,4769,2173,3084,4137, 915,3338,4411,4412,3339, // 4832 -1605,1835,5671,2748, 398,3656,4413,3926,4138, 328,1913,2860,4139,3927,1331,4414, // 4848 -3029, 937,4415,5672,3657,4140,4141,3424,2161,4770,3425, 524, 742, 538,3085,1012, // 4864 -5673,5674,3928,2466,5675, 658,1103, 225,3929,5676,5677,4771,5678,4772,5679,3267, // 4880 -1243,5680,4142, 963,2250,4773,5681,2714,3658,3186,5682,5683,2596,2332,5684,4774, // 4896 -5685,5686,5687,3536, 957,3426,2547,2033,1931,2941,2467, 870,2019,3659,1746,2780, // 4912 -2781,2439,2468,5688,3930,5689,3789,3130,3790,3537,3427,3791,5690,1179,3086,5691, // 4928 -3187,2378,4416,3792,2548,3188,3131,2749,4143,5692,3428,1556,2549,2297, 977,2901, // 4944 -2034,4144,1205,3429,5693,1765,3430,3189,2125,1271, 714,1689,4775,3538,5694,2333, // 4960 -3931, 533,4417,3660,2184, 617,5695,2469,3340,3539,2315,5696,5697,3190,5698,5699, // 4976 -3932,1988, 618, 427,2651,3540,3431,5700,5701,1244,1690,5702,2819,4418,4776,5703, // 4992 -3541,4777,5704,2284,1576, 473,3661,4419,3432, 972,5705,3662,5706,3087,5707,5708, // 5008 -4778,4779,5709,3793,4145,4146,5710, 153,4780, 356,5711,1892,2902,4420,2144, 408, // 5024 - 803,2357,5712,3933,5713,4421,1646,2578,2518,4781,4782,3934,5714,3935,4422,5715, // 5040 -2416,3433, 752,5716,5717,1962,3341,2979,5718, 746,3030,2470,4783,4423,3794, 698, // 5056 -4784,1893,4424,3663,2550,4785,3664,3936,5719,3191,3434,5720,1824,1302,4147,2715, // 5072 -3937,1974,4425,5721,4426,3192, 823,1303,1288,1236,2861,3542,4148,3435, 774,3938, // 5088 -5722,1581,4786,1304,2862,3939,4787,5723,2440,2162,1083,3268,4427,4149,4428, 344, // 5104 -1173, 288,2316, 454,1683,5724,5725,1461,4788,4150,2597,5726,5727,4789, 985, 894, // 5120 -5728,3436,3193,5729,1914,2942,3795,1989,5730,2111,1975,5731,4151,5732,2579,1194, // 5136 - 425,5733,4790,3194,1245,3796,4429,5734,5735,2863,5736, 636,4791,1856,3940, 760, // 5152 -1800,5737,4430,2212,1508,4792,4152,1894,1684,2298,5738,5739,4793,4431,4432,2213, // 5168 - 479,5740,5741, 832,5742,4153,2496,5743,2980,2497,3797, 990,3132, 627,1815,2652, // 5184 -4433,1582,4434,2126,2112,3543,4794,5744, 799,4435,3195,5745,4795,2113,1737,3031, // 5200 -1018, 543, 754,4436,3342,1676,4796,4797,4154,4798,1489,5746,3544,5747,2624,2903, // 5216 -4155,5748,5749,2981,5750,5751,5752,5753,3196,4799,4800,2185,1722,5754,3269,3270, // 5232 -1843,3665,1715, 481, 365,1976,1857,5755,5756,1963,2498,4801,5757,2127,3666,3271, // 5248 - 433,1895,2064,2076,5758, 602,2750,5759,5760,5761,5762,5763,3032,1628,3437,5764, // 5264 -3197,4802,4156,2904,4803,2519,5765,2551,2782,5766,5767,5768,3343,4804,2905,5769, // 5280 -4805,5770,2864,4806,4807,1221,2982,4157,2520,5771,5772,5773,1868,1990,5774,5775, // 5296 -5776,1896,5777,5778,4808,1897,4158, 318,5779,2095,4159,4437,5780,5781, 485,5782, // 5312 - 938,3941, 553,2680, 116,5783,3942,3667,5784,3545,2681,2783,3438,3344,2820,5785, // 5328 -3668,2943,4160,1747,2944,2983,5786,5787, 207,5788,4809,5789,4810,2521,5790,3033, // 5344 - 890,3669,3943,5791,1878,3798,3439,5792,2186,2358,3440,1652,5793,5794,5795, 941, // 5360 -2299, 208,3546,4161,2020, 330,4438,3944,2906,2499,3799,4439,4811,5796,5797,5798, // 5376 //last 512 - -/*************************************************************************************** - *Everything below is of no interest for detection purpose * - *************************************************************************************** - -2522,1613,4812,5799,3345,3945,2523,5800,4162,5801,1637,4163,2471,4813,3946,5802, // 5392 -2500,3034,3800,5803,5804,2195,4814,5805,2163,5806,5807,5808,5809,5810,5811,5812, // 5408 -5813,5814,5815,5816,5817,5818,5819,5820,5821,5822,5823,5824,5825,5826,5827,5828, // 5424 -5829,5830,5831,5832,5833,5834,5835,5836,5837,5838,5839,5840,5841,5842,5843,5844, // 5440 -5845,5846,5847,5848,5849,5850,5851,5852,5853,5854,5855,5856,5857,5858,5859,5860, // 5456 -5861,5862,5863,5864,5865,5866,5867,5868,5869,5870,5871,5872,5873,5874,5875,5876, // 5472 -5877,5878,5879,5880,5881,5882,5883,5884,5885,5886,5887,5888,5889,5890,5891,5892, // 5488 -5893,5894,5895,5896,5897,5898,5899,5900,5901,5902,5903,5904,5905,5906,5907,5908, // 5504 -5909,5910,5911,5912,5913,5914,5915,5916,5917,5918,5919,5920,5921,5922,5923,5924, // 5520 -5925,5926,5927,5928,5929,5930,5931,5932,5933,5934,5935,5936,5937,5938,5939,5940, // 5536 -5941,5942,5943,5944,5945,5946,5947,5948,5949,5950,5951,5952,5953,5954,5955,5956, // 5552 -5957,5958,5959,5960,5961,5962,5963,5964,5965,5966,5967,5968,5969,5970,5971,5972, // 5568 -5973,5974,5975,5976,5977,5978,5979,5980,5981,5982,5983,5984,5985,5986,5987,5988, // 5584 -5989,5990,5991,5992,5993,5994,5995,5996,5997,5998,5999,6000,6001,6002,6003,6004, // 5600 -6005,6006,6007,6008,6009,6010,6011,6012,6013,6014,6015,6016,6017,6018,6019,6020, // 5616 -6021,6022,6023,6024,6025,6026,6027,6028,6029,6030,6031,6032,6033,6034,6035,6036, // 5632 -6037,6038,6039,6040,6041,6042,6043,6044,6045,6046,6047,6048,6049,6050,6051,6052, // 5648 -6053,6054,6055,6056,6057,6058,6059,6060,6061,6062,6063,6064,6065,6066,6067,6068, // 5664 -6069,6070,6071,6072,6073,6074,6075,6076,6077,6078,6079,6080,6081,6082,6083,6084, // 5680 -6085,6086,6087,6088,6089,6090,6091,6092,6093,6094,6095,6096,6097,6098,6099,6100, // 5696 -6101,6102,6103,6104,6105,6106,6107,6108,6109,6110,6111,6112,6113,6114,6115,6116, // 5712 -6117,6118,6119,6120,6121,6122,6123,6124,6125,6126,6127,6128,6129,6130,6131,6132, // 5728 -6133,6134,6135,6136,6137,6138,6139,6140,6141,6142,6143,6144,6145,6146,6147,6148, // 5744 -6149,6150,6151,6152,6153,6154,6155,6156,6157,6158,6159,6160,6161,6162,6163,6164, // 5760 -6165,6166,6167,6168,6169,6170,6171,6172,6173,6174,6175,6176,6177,6178,6179,6180, // 5776 -6181,6182,6183,6184,6185,6186,6187,6188,6189,6190,6191,6192,6193,6194,6195,6196, // 5792 -6197,6198,6199,6200,6201,6202,6203,6204,6205,6206,6207,6208,6209,6210,6211,6212, // 5808 -6213,6214,6215,6216,6217,6218,6219,6220,6221,6222,6223,3670,6224,6225,6226,6227, // 5824 -6228,6229,6230,6231,6232,6233,6234,6235,6236,6237,6238,6239,6240,6241,6242,6243, // 5840 -6244,6245,6246,6247,6248,6249,6250,6251,6252,6253,6254,6255,6256,6257,6258,6259, // 5856 -6260,6261,6262,6263,6264,6265,6266,6267,6268,6269,6270,6271,6272,6273,6274,6275, // 5872 -6276,6277,6278,6279,6280,6281,6282,6283,6284,6285,4815,6286,6287,6288,6289,6290, // 5888 -6291,6292,4816,6293,6294,6295,6296,6297,6298,6299,6300,6301,6302,6303,6304,6305, // 5904 -6306,6307,6308,6309,6310,6311,4817,4818,6312,6313,6314,6315,6316,6317,6318,4819, // 5920 -6319,6320,6321,6322,6323,6324,6325,6326,6327,6328,6329,6330,6331,6332,6333,6334, // 5936 -6335,6336,6337,4820,6338,6339,6340,6341,6342,6343,6344,6345,6346,6347,6348,6349, // 5952 -6350,6351,6352,6353,6354,6355,6356,6357,6358,6359,6360,6361,6362,6363,6364,6365, // 5968 -6366,6367,6368,6369,6370,6371,6372,6373,6374,6375,6376,6377,6378,6379,6380,6381, // 5984 -6382,6383,6384,6385,6386,6387,6388,6389,6390,6391,6392,6393,6394,6395,6396,6397, // 6000 -6398,6399,6400,6401,6402,6403,6404,6405,6406,6407,6408,6409,6410,3441,6411,6412, // 6016 -6413,6414,6415,6416,6417,6418,6419,6420,6421,6422,6423,6424,6425,4440,6426,6427, // 6032 -6428,6429,6430,6431,6432,6433,6434,6435,6436,6437,6438,6439,6440,6441,6442,6443, // 6048 -6444,6445,6446,6447,6448,6449,6450,6451,6452,6453,6454,4821,6455,6456,6457,6458, // 6064 -6459,6460,6461,6462,6463,6464,6465,6466,6467,6468,6469,6470,6471,6472,6473,6474, // 6080 -6475,6476,6477,3947,3948,6478,6479,6480,6481,3272,4441,6482,6483,6484,6485,4442, // 6096 -6486,6487,6488,6489,6490,6491,6492,6493,6494,6495,6496,4822,6497,6498,6499,6500, // 6112 -6501,6502,6503,6504,6505,6506,6507,6508,6509,6510,6511,6512,6513,6514,6515,6516, // 6128 -6517,6518,6519,6520,6521,6522,6523,6524,6525,6526,6527,6528,6529,6530,6531,6532, // 6144 -6533,6534,6535,6536,6537,6538,6539,6540,6541,6542,6543,6544,6545,6546,6547,6548, // 6160 -6549,6550,6551,6552,6553,6554,6555,6556,2784,6557,4823,6558,6559,6560,6561,6562, // 6176 -6563,6564,6565,6566,6567,6568,6569,3949,6570,6571,6572,4824,6573,6574,6575,6576, // 6192 -6577,6578,6579,6580,6581,6582,6583,4825,6584,6585,6586,3950,2785,6587,6588,6589, // 6208 -6590,6591,6592,6593,6594,6595,6596,6597,6598,6599,6600,6601,6602,6603,6604,6605, // 6224 -6606,6607,6608,6609,6610,6611,6612,4826,6613,6614,6615,4827,6616,6617,6618,6619, // 6240 -6620,6621,6622,6623,6624,6625,4164,6626,6627,6628,6629,6630,6631,6632,6633,6634, // 6256 -3547,6635,4828,6636,6637,6638,6639,6640,6641,6642,3951,2984,6643,6644,6645,6646, // 6272 -6647,6648,6649,4165,6650,4829,6651,6652,4830,6653,6654,6655,6656,6657,6658,6659, // 6288 -6660,6661,6662,4831,6663,6664,6665,6666,6667,6668,6669,6670,6671,4166,6672,4832, // 6304 -3952,6673,6674,6675,6676,4833,6677,6678,6679,4167,6680,6681,6682,3198,6683,6684, // 6320 -6685,6686,6687,6688,6689,6690,6691,6692,6693,6694,6695,6696,6697,4834,6698,6699, // 6336 -6700,6701,6702,6703,6704,6705,6706,6707,6708,6709,6710,6711,6712,6713,6714,6715, // 6352 -6716,6717,6718,6719,6720,6721,6722,6723,6724,6725,6726,6727,6728,6729,6730,6731, // 6368 -6732,6733,6734,4443,6735,6736,6737,6738,6739,6740,6741,6742,6743,6744,6745,4444, // 6384 -6746,6747,6748,6749,6750,6751,6752,6753,6754,6755,6756,6757,6758,6759,6760,6761, // 6400 -6762,6763,6764,6765,6766,6767,6768,6769,6770,6771,6772,6773,6774,6775,6776,6777, // 6416 -6778,6779,6780,6781,4168,6782,6783,3442,6784,6785,6786,6787,6788,6789,6790,6791, // 6432 -4169,6792,6793,6794,6795,6796,6797,6798,6799,6800,6801,6802,6803,6804,6805,6806, // 6448 -6807,6808,6809,6810,6811,4835,6812,6813,6814,4445,6815,6816,4446,6817,6818,6819, // 6464 -6820,6821,6822,6823,6824,6825,6826,6827,6828,6829,6830,6831,6832,6833,6834,6835, // 6480 -3548,6836,6837,6838,6839,6840,6841,6842,6843,6844,6845,6846,4836,6847,6848,6849, // 6496 -6850,6851,6852,6853,6854,3953,6855,6856,6857,6858,6859,6860,6861,6862,6863,6864, // 6512 -6865,6866,6867,6868,6869,6870,6871,6872,6873,6874,6875,6876,6877,3199,6878,6879, // 6528 -6880,6881,6882,4447,6883,6884,6885,6886,6887,6888,6889,6890,6891,6892,6893,6894, // 6544 -6895,6896,6897,6898,6899,6900,6901,6902,6903,6904,4170,6905,6906,6907,6908,6909, // 6560 -6910,6911,6912,6913,6914,6915,6916,6917,6918,6919,6920,6921,6922,6923,6924,6925, // 6576 -6926,6927,4837,6928,6929,6930,6931,6932,6933,6934,6935,6936,3346,6937,6938,4838, // 6592 -6939,6940,6941,4448,6942,6943,6944,6945,6946,4449,6947,6948,6949,6950,6951,6952, // 6608 -6953,6954,6955,6956,6957,6958,6959,6960,6961,6962,6963,6964,6965,6966,6967,6968, // 6624 -6969,6970,6971,6972,6973,6974,6975,6976,6977,6978,6979,6980,6981,6982,6983,6984, // 6640 -6985,6986,6987,6988,6989,6990,6991,6992,6993,6994,3671,6995,6996,6997,6998,4839, // 6656 -6999,7000,7001,7002,3549,7003,7004,7005,7006,7007,7008,7009,7010,7011,7012,7013, // 6672 -7014,7015,7016,7017,7018,7019,7020,7021,7022,7023,7024,7025,7026,7027,7028,7029, // 6688 -7030,4840,7031,7032,7033,7034,7035,7036,7037,7038,4841,7039,7040,7041,7042,7043, // 6704 -7044,7045,7046,7047,7048,7049,7050,7051,7052,7053,7054,7055,7056,7057,7058,7059, // 6720 -7060,7061,7062,7063,7064,7065,7066,7067,7068,7069,7070,2985,7071,7072,7073,7074, // 6736 -7075,7076,7077,7078,7079,7080,4842,7081,7082,7083,7084,7085,7086,7087,7088,7089, // 6752 -7090,7091,7092,7093,7094,7095,7096,7097,7098,7099,7100,7101,7102,7103,7104,7105, // 6768 -7106,7107,7108,7109,7110,7111,7112,7113,7114,7115,7116,7117,7118,4450,7119,7120, // 6784 -7121,7122,7123,7124,7125,7126,7127,7128,7129,7130,7131,7132,7133,7134,7135,7136, // 6800 -7137,7138,7139,7140,7141,7142,7143,4843,7144,7145,7146,7147,7148,7149,7150,7151, // 6816 -7152,7153,7154,7155,7156,7157,7158,7159,7160,7161,7162,7163,7164,7165,7166,7167, // 6832 -7168,7169,7170,7171,7172,7173,7174,7175,7176,7177,7178,7179,7180,7181,7182,7183, // 6848 -7184,7185,7186,7187,7188,4171,4172,7189,7190,7191,7192,7193,7194,7195,7196,7197, // 6864 -7198,7199,7200,7201,7202,7203,7204,7205,7206,7207,7208,7209,7210,7211,7212,7213, // 6880 -7214,7215,7216,7217,7218,7219,7220,7221,7222,7223,7224,7225,7226,7227,7228,7229, // 6896 -7230,7231,7232,7233,7234,7235,7236,7237,7238,7239,7240,7241,7242,7243,7244,7245, // 6912 -7246,7247,7248,7249,7250,7251,7252,7253,7254,7255,7256,7257,7258,7259,7260,7261, // 6928 -7262,7263,7264,7265,7266,7267,7268,7269,7270,7271,7272,7273,7274,7275,7276,7277, // 6944 -7278,7279,7280,7281,7282,7283,7284,7285,7286,7287,7288,7289,7290,7291,7292,7293, // 6960 -7294,7295,7296,4844,7297,7298,7299,7300,7301,7302,7303,7304,7305,7306,7307,7308, // 6976 -7309,7310,7311,7312,7313,7314,7315,7316,4451,7317,7318,7319,7320,7321,7322,7323, // 6992 -7324,7325,7326,7327,7328,7329,7330,7331,7332,7333,7334,7335,7336,7337,7338,7339, // 7008 -7340,7341,7342,7343,7344,7345,7346,7347,7348,7349,7350,7351,7352,7353,4173,7354, // 7024 -7355,4845,7356,7357,7358,7359,7360,7361,7362,7363,7364,7365,7366,7367,7368,7369, // 7040 -7370,7371,7372,7373,7374,7375,7376,7377,7378,7379,7380,7381,7382,7383,7384,7385, // 7056 -7386,7387,7388,4846,7389,7390,7391,7392,7393,7394,7395,7396,7397,7398,7399,7400, // 7072 -7401,7402,7403,7404,7405,3672,7406,7407,7408,7409,7410,7411,7412,7413,7414,7415, // 7088 -7416,7417,7418,7419,7420,7421,7422,7423,7424,7425,7426,7427,7428,7429,7430,7431, // 7104 -7432,7433,7434,7435,7436,7437,7438,7439,7440,7441,7442,7443,7444,7445,7446,7447, // 7120 -7448,7449,7450,7451,7452,7453,4452,7454,3200,7455,7456,7457,7458,7459,7460,7461, // 7136 -7462,7463,7464,7465,7466,7467,7468,7469,7470,7471,7472,7473,7474,4847,7475,7476, // 7152 -7477,3133,7478,7479,7480,7481,7482,7483,7484,7485,7486,7487,7488,7489,7490,7491, // 7168 -7492,7493,7494,7495,7496,7497,7498,7499,7500,7501,7502,3347,7503,7504,7505,7506, // 7184 -7507,7508,7509,7510,7511,7512,7513,7514,7515,7516,7517,7518,7519,7520,7521,4848, // 7200 -7522,7523,7524,7525,7526,7527,7528,7529,7530,7531,7532,7533,7534,7535,7536,7537, // 7216 -7538,7539,7540,7541,7542,7543,7544,7545,7546,7547,7548,7549,3801,4849,7550,7551, // 7232 -7552,7553,7554,7555,7556,7557,7558,7559,7560,7561,7562,7563,7564,7565,7566,7567, // 7248 -7568,7569,3035,7570,7571,7572,7573,7574,7575,7576,7577,7578,7579,7580,7581,7582, // 7264 -7583,7584,7585,7586,7587,7588,7589,7590,7591,7592,7593,7594,7595,7596,7597,7598, // 7280 -7599,7600,7601,7602,7603,7604,7605,7606,7607,7608,7609,7610,7611,7612,7613,7614, // 7296 -7615,7616,4850,7617,7618,3802,7619,7620,7621,7622,7623,7624,7625,7626,7627,7628, // 7312 -7629,7630,7631,7632,4851,7633,7634,7635,7636,7637,7638,7639,7640,7641,7642,7643, // 7328 -7644,7645,7646,7647,7648,7649,7650,7651,7652,7653,7654,7655,7656,7657,7658,7659, // 7344 -7660,7661,7662,7663,7664,7665,7666,7667,7668,7669,7670,4453,7671,7672,7673,7674, // 7360 -7675,7676,7677,7678,7679,7680,7681,7682,7683,7684,7685,7686,7687,7688,7689,7690, // 7376 -7691,7692,7693,7694,7695,7696,7697,3443,7698,7699,7700,7701,7702,4454,7703,7704, // 7392 -7705,7706,7707,7708,7709,7710,7711,7712,7713,2472,7714,7715,7716,7717,7718,7719, // 7408 -7720,7721,7722,7723,7724,7725,7726,7727,7728,7729,7730,7731,3954,7732,7733,7734, // 7424 -7735,7736,7737,7738,7739,7740,7741,7742,7743,7744,7745,7746,7747,7748,7749,7750, // 7440 -3134,7751,7752,4852,7753,7754,7755,4853,7756,7757,7758,7759,7760,4174,7761,7762, // 7456 -7763,7764,7765,7766,7767,7768,7769,7770,7771,7772,7773,7774,7775,7776,7777,7778, // 7472 -7779,7780,7781,7782,7783,7784,7785,7786,7787,7788,7789,7790,7791,7792,7793,7794, // 7488 -7795,7796,7797,7798,7799,7800,7801,7802,7803,7804,7805,4854,7806,7807,7808,7809, // 7504 -7810,7811,7812,7813,7814,7815,7816,7817,7818,7819,7820,7821,7822,7823,7824,7825, // 7520 -4855,7826,7827,7828,7829,7830,7831,7832,7833,7834,7835,7836,7837,7838,7839,7840, // 7536 -7841,7842,7843,7844,7845,7846,7847,3955,7848,7849,7850,7851,7852,7853,7854,7855, // 7552 -7856,7857,7858,7859,7860,3444,7861,7862,7863,7864,7865,7866,7867,7868,7869,7870, // 7568 -7871,7872,7873,7874,7875,7876,7877,7878,7879,7880,7881,7882,7883,7884,7885,7886, // 7584 -7887,7888,7889,7890,7891,4175,7892,7893,7894,7895,7896,4856,4857,7897,7898,7899, // 7600 -7900,2598,7901,7902,7903,7904,7905,7906,7907,7908,4455,7909,7910,7911,7912,7913, // 7616 -7914,3201,7915,7916,7917,7918,7919,7920,7921,4858,7922,7923,7924,7925,7926,7927, // 7632 -7928,7929,7930,7931,7932,7933,7934,7935,7936,7937,7938,7939,7940,7941,7942,7943, // 7648 -7944,7945,7946,7947,7948,7949,7950,7951,7952,7953,7954,7955,7956,7957,7958,7959, // 7664 -7960,7961,7962,7963,7964,7965,7966,7967,7968,7969,7970,7971,7972,7973,7974,7975, // 7680 -7976,7977,7978,7979,7980,7981,4859,7982,7983,7984,7985,7986,7987,7988,7989,7990, // 7696 -7991,7992,7993,7994,7995,7996,4860,7997,7998,7999,8000,8001,8002,8003,8004,8005, // 7712 -8006,8007,8008,8009,8010,8011,8012,8013,8014,8015,8016,4176,8017,8018,8019,8020, // 7728 -8021,8022,8023,4861,8024,8025,8026,8027,8028,8029,8030,8031,8032,8033,8034,8035, // 7744 -8036,4862,4456,8037,8038,8039,8040,4863,8041,8042,8043,8044,8045,8046,8047,8048, // 7760 -8049,8050,8051,8052,8053,8054,8055,8056,8057,8058,8059,8060,8061,8062,8063,8064, // 7776 -8065,8066,8067,8068,8069,8070,8071,8072,8073,8074,8075,8076,8077,8078,8079,8080, // 7792 -8081,8082,8083,8084,8085,8086,8087,8088,8089,8090,8091,8092,8093,8094,8095,8096, // 7808 -8097,8098,8099,4864,4177,8100,8101,8102,8103,8104,8105,8106,8107,8108,8109,8110, // 7824 -8111,8112,8113,8114,8115,8116,8117,8118,8119,8120,4178,8121,8122,8123,8124,8125, // 7840 -8126,8127,8128,8129,8130,8131,8132,8133,8134,8135,8136,8137,8138,8139,8140,8141, // 7856 -8142,8143,8144,8145,4865,4866,8146,8147,8148,8149,8150,8151,8152,8153,8154,8155, // 7872 -8156,8157,8158,8159,8160,8161,8162,8163,8164,8165,4179,8166,8167,8168,8169,8170, // 7888 -8171,8172,8173,8174,8175,8176,8177,8178,8179,8180,8181,4457,8182,8183,8184,8185, // 7904 -8186,8187,8188,8189,8190,8191,8192,8193,8194,8195,8196,8197,8198,8199,8200,8201, // 7920 -8202,8203,8204,8205,8206,8207,8208,8209,8210,8211,8212,8213,8214,8215,8216,8217, // 7936 -8218,8219,8220,8221,8222,8223,8224,8225,8226,8227,8228,8229,8230,8231,8232,8233, // 7952 -8234,8235,8236,8237,8238,8239,8240,8241,8242,8243,8244,8245,8246,8247,8248,8249, // 7968 -8250,8251,8252,8253,8254,8255,8256,3445,8257,8258,8259,8260,8261,8262,4458,8263, // 7984 -8264,8265,8266,8267,8268,8269,8270,8271,8272,4459,8273,8274,8275,8276,3550,8277, // 8000 -8278,8279,8280,8281,8282,8283,8284,8285,8286,8287,8288,8289,4460,8290,8291,8292, // 8016 -8293,8294,8295,8296,8297,8298,8299,8300,8301,8302,8303,8304,8305,8306,8307,4867, // 8032 -8308,8309,8310,8311,8312,3551,8313,8314,8315,8316,8317,8318,8319,8320,8321,8322, // 8048 -8323,8324,8325,8326,4868,8327,8328,8329,8330,8331,8332,8333,8334,8335,8336,8337, // 8064 -8338,8339,8340,8341,8342,8343,8344,8345,8346,8347,8348,8349,8350,8351,8352,8353, // 8080 -8354,8355,8356,8357,8358,8359,8360,8361,8362,8363,4869,4461,8364,8365,8366,8367, // 8096 -8368,8369,8370,4870,8371,8372,8373,8374,8375,8376,8377,8378,8379,8380,8381,8382, // 8112 -8383,8384,8385,8386,8387,8388,8389,8390,8391,8392,8393,8394,8395,8396,8397,8398, // 8128 -8399,8400,8401,8402,8403,8404,8405,8406,8407,8408,8409,8410,4871,8411,8412,8413, // 8144 -8414,8415,8416,8417,8418,8419,8420,8421,8422,4462,8423,8424,8425,8426,8427,8428, // 8160 -8429,8430,8431,8432,8433,2986,8434,8435,8436,8437,8438,8439,8440,8441,8442,8443, // 8176 -8444,8445,8446,8447,8448,8449,8450,8451,8452,8453,8454,8455,8456,8457,8458,8459, // 8192 -8460,8461,8462,8463,8464,8465,8466,8467,8468,8469,8470,8471,8472,8473,8474,8475, // 8208 -8476,8477,8478,4180,8479,8480,8481,8482,8483,8484,8485,8486,8487,8488,8489,8490, // 8224 -8491,8492,8493,8494,8495,8496,8497,8498,8499,8500,8501,8502,8503,8504,8505,8506, // 8240 -8507,8508,8509,8510,8511,8512,8513,8514,8515,8516,8517,8518,8519,8520,8521,8522, // 8256 -8523,8524,8525,8526,8527,8528,8529,8530,8531,8532,8533,8534,8535,8536,8537,8538, // 8272 -8539,8540,8541,8542,8543,8544,8545,8546,8547,8548,8549,8550,8551,8552,8553,8554, // 8288 -8555,8556,8557,8558,8559,8560,8561,8562,8563,8564,4872,8565,8566,8567,8568,8569, // 8304 -8570,8571,8572,8573,4873,8574,8575,8576,8577,8578,8579,8580,8581,8582,8583,8584, // 8320 -8585,8586,8587,8588,8589,8590,8591,8592,8593,8594,8595,8596,8597,8598,8599,8600, // 8336 -8601,8602,8603,8604,8605,3803,8606,8607,8608,8609,8610,8611,8612,8613,4874,3804, // 8352 -8614,8615,8616,8617,8618,8619,8620,8621,3956,8622,8623,8624,8625,8626,8627,8628, // 8368 -8629,8630,8631,8632,8633,8634,8635,8636,8637,8638,2865,8639,8640,8641,8642,8643, // 8384 -8644,8645,8646,8647,8648,8649,8650,8651,8652,8653,8654,8655,8656,4463,8657,8658, // 8400 -8659,4875,4876,8660,8661,8662,8663,8664,8665,8666,8667,8668,8669,8670,8671,8672, // 8416 -8673,8674,8675,8676,8677,8678,8679,8680,8681,4464,8682,8683,8684,8685,8686,8687, // 8432 -8688,8689,8690,8691,8692,8693,8694,8695,8696,8697,8698,8699,8700,8701,8702,8703, // 8448 -8704,8705,8706,8707,8708,8709,2261,8710,8711,8712,8713,8714,8715,8716,8717,8718, // 8464 -8719,8720,8721,8722,8723,8724,8725,8726,8727,8728,8729,8730,8731,8732,8733,4181, // 8480 -8734,8735,8736,8737,8738,8739,8740,8741,8742,8743,8744,8745,8746,8747,8748,8749, // 8496 -8750,8751,8752,8753,8754,8755,8756,8757,8758,8759,8760,8761,8762,8763,4877,8764, // 8512 -8765,8766,8767,8768,8769,8770,8771,8772,8773,8774,8775,8776,8777,8778,8779,8780, // 8528 -8781,8782,8783,8784,8785,8786,8787,8788,4878,8789,4879,8790,8791,8792,4880,8793, // 8544 -8794,8795,8796,8797,8798,8799,8800,8801,4881,8802,8803,8804,8805,8806,8807,8808, // 8560 -8809,8810,8811,8812,8813,8814,8815,3957,8816,8817,8818,8819,8820,8821,8822,8823, // 8576 -8824,8825,8826,8827,8828,8829,8830,8831,8832,8833,8834,8835,8836,8837,8838,8839, // 8592 -8840,8841,8842,8843,8844,8845,8846,8847,4882,8848,8849,8850,8851,8852,8853,8854, // 8608 -8855,8856,8857,8858,8859,8860,8861,8862,8863,8864,8865,8866,8867,8868,8869,8870, // 8624 -8871,8872,8873,8874,8875,8876,8877,8878,8879,8880,8881,8882,8883,8884,3202,8885, // 8640 -8886,8887,8888,8889,8890,8891,8892,8893,8894,8895,8896,8897,8898,8899,8900,8901, // 8656 -8902,8903,8904,8905,8906,8907,8908,8909,8910,8911,8912,8913,8914,8915,8916,8917, // 8672 -8918,8919,8920,8921,8922,8923,8924,4465,8925,8926,8927,8928,8929,8930,8931,8932, // 8688 -4883,8933,8934,8935,8936,8937,8938,8939,8940,8941,8942,8943,2214,8944,8945,8946, // 8704 -8947,8948,8949,8950,8951,8952,8953,8954,8955,8956,8957,8958,8959,8960,8961,8962, // 8720 -8963,8964,8965,4884,8966,8967,8968,8969,8970,8971,8972,8973,8974,8975,8976,8977, // 8736 -8978,8979,8980,8981,8982,8983,8984,8985,8986,8987,8988,8989,8990,8991,8992,4885, // 8752 -8993,8994,8995,8996,8997,8998,8999,9000,9001,9002,9003,9004,9005,9006,9007,9008, // 8768 -9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,4182,9022,9023, // 8784 -9024,9025,9026,9027,9028,9029,9030,9031,9032,9033,9034,9035,9036,9037,9038,9039, // 8800 -9040,9041,9042,9043,9044,9045,9046,9047,9048,9049,9050,9051,9052,9053,9054,9055, // 8816 -9056,9057,9058,9059,9060,9061,9062,9063,4886,9064,9065,9066,9067,9068,9069,4887, // 8832 -9070,9071,9072,9073,9074,9075,9076,9077,9078,9079,9080,9081,9082,9083,9084,9085, // 8848 -9086,9087,9088,9089,9090,9091,9092,9093,9094,9095,9096,9097,9098,9099,9100,9101, // 8864 -9102,9103,9104,9105,9106,9107,9108,9109,9110,9111,9112,9113,9114,9115,9116,9117, // 8880 -9118,9119,9120,9121,9122,9123,9124,9125,9126,9127,9128,9129,9130,9131,9132,9133, // 8896 -9134,9135,9136,9137,9138,9139,9140,9141,3958,9142,9143,9144,9145,9146,9147,9148, // 8912 -9149,9150,9151,4888,9152,9153,9154,9155,9156,9157,9158,9159,9160,9161,9162,9163, // 8928 -9164,9165,9166,9167,9168,9169,9170,9171,9172,9173,9174,9175,4889,9176,9177,9178, // 8944 -9179,9180,9181,9182,9183,9184,9185,9186,9187,9188,9189,9190,9191,9192,9193,9194, // 8960 -9195,9196,9197,9198,9199,9200,9201,9202,9203,4890,9204,9205,9206,9207,9208,9209, // 8976 -9210,9211,9212,9213,9214,9215,9216,9217,9218,9219,9220,9221,9222,4466,9223,9224, // 8992 -9225,9226,9227,9228,9229,9230,9231,9232,9233,9234,9235,9236,9237,9238,9239,9240, // 9008 -9241,9242,9243,9244,9245,4891,9246,9247,9248,9249,9250,9251,9252,9253,9254,9255, // 9024 -9256,9257,4892,9258,9259,9260,9261,4893,4894,9262,9263,9264,9265,9266,9267,9268, // 9040 -9269,9270,9271,9272,9273,4467,9274,9275,9276,9277,9278,9279,9280,9281,9282,9283, // 9056 -9284,9285,3673,9286,9287,9288,9289,9290,9291,9292,9293,9294,9295,9296,9297,9298, // 9072 -9299,9300,9301,9302,9303,9304,9305,9306,9307,9308,9309,9310,9311,9312,9313,9314, // 9088 -9315,9316,9317,9318,9319,9320,9321,9322,4895,9323,9324,9325,9326,9327,9328,9329, // 9104 -9330,9331,9332,9333,9334,9335,9336,9337,9338,9339,9340,9341,9342,9343,9344,9345, // 9120 -9346,9347,4468,9348,9349,9350,9351,9352,9353,9354,9355,9356,9357,9358,9359,9360, // 9136 -9361,9362,9363,9364,9365,9366,9367,9368,9369,9370,9371,9372,9373,4896,9374,4469, // 9152 -9375,9376,9377,9378,9379,4897,9380,9381,9382,9383,9384,9385,9386,9387,9388,9389, // 9168 -9390,9391,9392,9393,9394,9395,9396,9397,9398,9399,9400,9401,9402,9403,9404,9405, // 9184 -9406,4470,9407,2751,9408,9409,3674,3552,9410,9411,9412,9413,9414,9415,9416,9417, // 9200 -9418,9419,9420,9421,4898,9422,9423,9424,9425,9426,9427,9428,9429,3959,9430,9431, // 9216 -9432,9433,9434,9435,9436,4471,9437,9438,9439,9440,9441,9442,9443,9444,9445,9446, // 9232 -9447,9448,9449,9450,3348,9451,9452,9453,9454,9455,9456,9457,9458,9459,9460,9461, // 9248 -9462,9463,9464,9465,9466,9467,9468,9469,9470,9471,9472,4899,9473,9474,9475,9476, // 9264 -9477,4900,9478,9479,9480,9481,9482,9483,9484,9485,9486,9487,9488,3349,9489,9490, // 9280 -9491,9492,9493,9494,9495,9496,9497,9498,9499,9500,9501,9502,9503,9504,9505,9506, // 9296 -9507,9508,9509,9510,9511,9512,9513,9514,9515,9516,9517,9518,9519,9520,4901,9521, // 9312 -9522,9523,9524,9525,9526,4902,9527,9528,9529,9530,9531,9532,9533,9534,9535,9536, // 9328 -9537,9538,9539,9540,9541,9542,9543,9544,9545,9546,9547,9548,9549,9550,9551,9552, // 9344 -9553,9554,9555,9556,9557,9558,9559,9560,9561,9562,9563,9564,9565,9566,9567,9568, // 9360 -9569,9570,9571,9572,9573,9574,9575,9576,9577,9578,9579,9580,9581,9582,9583,9584, // 9376 -3805,9585,9586,9587,9588,9589,9590,9591,9592,9593,9594,9595,9596,9597,9598,9599, // 9392 -9600,9601,9602,4903,9603,9604,9605,9606,9607,4904,9608,9609,9610,9611,9612,9613, // 9408 -9614,4905,9615,9616,9617,9618,9619,9620,9621,9622,9623,9624,9625,9626,9627,9628, // 9424 -9629,9630,9631,9632,4906,9633,9634,9635,9636,9637,9638,9639,9640,9641,9642,9643, // 9440 -4907,9644,9645,9646,9647,9648,9649,9650,9651,9652,9653,9654,9655,9656,9657,9658, // 9456 -9659,9660,9661,9662,9663,9664,9665,9666,9667,9668,9669,9670,9671,9672,4183,9673, // 9472 -9674,9675,9676,9677,4908,9678,9679,9680,9681,4909,9682,9683,9684,9685,9686,9687, // 9488 -9688,9689,9690,4910,9691,9692,9693,3675,9694,9695,9696,2945,9697,9698,9699,9700, // 9504 -9701,9702,9703,9704,9705,4911,9706,9707,9708,9709,9710,9711,9712,9713,9714,9715, // 9520 -9716,9717,9718,9719,9720,9721,9722,9723,9724,9725,9726,9727,9728,9729,9730,9731, // 9536 -9732,9733,9734,9735,4912,9736,9737,9738,9739,9740,4913,9741,9742,9743,9744,9745, // 9552 -9746,9747,9748,9749,9750,9751,9752,9753,9754,9755,9756,9757,9758,4914,9759,9760, // 9568 -9761,9762,9763,9764,9765,9766,9767,9768,9769,9770,9771,9772,9773,9774,9775,9776, // 9584 -9777,9778,9779,9780,9781,9782,4915,9783,9784,9785,9786,9787,9788,9789,9790,9791, // 9600 -9792,9793,4916,9794,9795,9796,9797,9798,9799,9800,9801,9802,9803,9804,9805,9806, // 9616 -9807,9808,9809,9810,9811,9812,9813,9814,9815,9816,9817,9818,9819,9820,9821,9822, // 9632 -9823,9824,9825,9826,9827,9828,9829,9830,9831,9832,9833,9834,9835,9836,9837,9838, // 9648 -9839,9840,9841,9842,9843,9844,9845,9846,9847,9848,9849,9850,9851,9852,9853,9854, // 9664 -9855,9856,9857,9858,9859,9860,9861,9862,9863,9864,9865,9866,9867,9868,4917,9869, // 9680 -9870,9871,9872,9873,9874,9875,9876,9877,9878,9879,9880,9881,9882,9883,9884,9885, // 9696 -9886,9887,9888,9889,9890,9891,9892,4472,9893,9894,9895,9896,9897,3806,9898,9899, // 9712 -9900,9901,9902,9903,9904,9905,9906,9907,9908,9909,9910,9911,9912,9913,9914,4918, // 9728 -9915,9916,9917,4919,9918,9919,9920,9921,4184,9922,9923,9924,9925,9926,9927,9928, // 9744 -9929,9930,9931,9932,9933,9934,9935,9936,9937,9938,9939,9940,9941,9942,9943,9944, // 9760 -9945,9946,4920,9947,9948,9949,9950,9951,9952,9953,9954,9955,4185,9956,9957,9958, // 9776 -9959,9960,9961,9962,9963,9964,9965,4921,9966,9967,9968,4473,9969,9970,9971,9972, // 9792 -9973,9974,9975,9976,9977,4474,9978,9979,9980,9981,9982,9983,9984,9985,9986,9987, // 9808 -9988,9989,9990,9991,9992,9993,9994,9995,9996,9997,9998,9999,10000,10001,10002,10003, // 9824 -10004,10005,10006,10007,10008,10009,10010,10011,10012,10013,10014,10015,10016,10017,10018,10019, // 9840 -10020,10021,4922,10022,4923,10023,10024,10025,10026,10027,10028,10029,10030,10031,10032,10033, // 9856 -10034,10035,10036,10037,10038,10039,10040,10041,10042,10043,10044,10045,10046,10047,10048,4924, // 9872 -10049,10050,10051,10052,10053,10054,10055,10056,10057,10058,10059,10060,10061,10062,10063,10064, // 9888 -10065,10066,10067,10068,10069,10070,10071,10072,10073,10074,10075,10076,10077,10078,10079,10080, // 9904 -10081,10082,10083,10084,10085,10086,10087,4475,10088,10089,10090,10091,10092,10093,10094,10095, // 9920 -10096,10097,4476,10098,10099,10100,10101,10102,10103,10104,10105,10106,10107,10108,10109,10110, // 9936 -10111,2174,10112,10113,10114,10115,10116,10117,10118,10119,10120,10121,10122,10123,10124,10125, // 9952 -10126,10127,10128,10129,10130,10131,10132,10133,10134,10135,10136,10137,10138,10139,10140,3807, // 9968 -4186,4925,10141,10142,10143,10144,10145,10146,10147,4477,4187,10148,10149,10150,10151,10152, // 9984 -10153,4188,10154,10155,10156,10157,10158,10159,10160,10161,4926,10162,10163,10164,10165,10166, //10000 -10167,10168,10169,10170,10171,10172,10173,10174,10175,10176,10177,10178,10179,10180,10181,10182, //10016 -10183,10184,10185,10186,10187,10188,10189,10190,10191,10192,3203,10193,10194,10195,10196,10197, //10032 -10198,10199,10200,4478,10201,10202,10203,10204,4479,10205,10206,10207,10208,10209,10210,10211, //10048 -10212,10213,10214,10215,10216,10217,10218,10219,10220,10221,10222,10223,10224,10225,10226,10227, //10064 -10228,10229,10230,10231,10232,10233,10234,4927,10235,10236,10237,10238,10239,10240,10241,10242, //10080 -10243,10244,10245,10246,10247,10248,10249,10250,10251,10252,10253,10254,10255,10256,10257,10258, //10096 -10259,10260,10261,10262,10263,10264,10265,10266,10267,10268,10269,10270,10271,10272,10273,4480, //10112 -4928,4929,10274,10275,10276,10277,10278,10279,10280,10281,10282,10283,10284,10285,10286,10287, //10128 -10288,10289,10290,10291,10292,10293,10294,10295,10296,10297,10298,10299,10300,10301,10302,10303, //10144 -10304,10305,10306,10307,10308,10309,10310,10311,10312,10313,10314,10315,10316,10317,10318,10319, //10160 -10320,10321,10322,10323,10324,10325,10326,10327,10328,10329,10330,10331,10332,10333,10334,4930, //10176 -10335,10336,10337,10338,10339,10340,10341,10342,4931,10343,10344,10345,10346,10347,10348,10349, //10192 -10350,10351,10352,10353,10354,10355,3088,10356,2786,10357,10358,10359,10360,4189,10361,10362, //10208 -10363,10364,10365,10366,10367,10368,10369,10370,10371,10372,10373,10374,10375,4932,10376,10377, //10224 -10378,10379,10380,10381,10382,10383,10384,10385,10386,10387,10388,10389,10390,10391,10392,4933, //10240 -10393,10394,10395,4934,10396,10397,10398,10399,10400,10401,10402,10403,10404,10405,10406,10407, //10256 -10408,10409,10410,10411,10412,3446,10413,10414,10415,10416,10417,10418,10419,10420,10421,10422, //10272 -10423,4935,10424,10425,10426,10427,10428,10429,10430,4936,10431,10432,10433,10434,10435,10436, //10288 -10437,10438,10439,10440,10441,10442,10443,4937,10444,10445,10446,10447,4481,10448,10449,10450, //10304 -10451,10452,10453,10454,10455,10456,10457,10458,10459,10460,10461,10462,10463,10464,10465,10466, //10320 -10467,10468,10469,10470,10471,10472,10473,10474,10475,10476,10477,10478,10479,10480,10481,10482, //10336 -10483,10484,10485,10486,10487,10488,10489,10490,10491,10492,10493,10494,10495,10496,10497,10498, //10352 -10499,10500,10501,10502,10503,10504,10505,4938,10506,10507,10508,10509,10510,2552,10511,10512, //10368 -10513,10514,10515,10516,3447,10517,10518,10519,10520,10521,10522,10523,10524,10525,10526,10527, //10384 -10528,10529,10530,10531,10532,10533,10534,10535,10536,10537,10538,10539,10540,10541,10542,10543, //10400 -4482,10544,4939,10545,10546,10547,10548,10549,10550,10551,10552,10553,10554,10555,10556,10557, //10416 -10558,10559,10560,10561,10562,10563,10564,10565,10566,10567,3676,4483,10568,10569,10570,10571, //10432 -10572,3448,10573,10574,10575,10576,10577,10578,10579,10580,10581,10582,10583,10584,10585,10586, //10448 -10587,10588,10589,10590,10591,10592,10593,10594,10595,10596,10597,10598,10599,10600,10601,10602, //10464 -10603,10604,10605,10606,10607,10608,10609,10610,10611,10612,10613,10614,10615,10616,10617,10618, //10480 -10619,10620,10621,10622,10623,10624,10625,10626,10627,4484,10628,10629,10630,10631,10632,4940, //10496 -10633,10634,10635,10636,10637,10638,10639,10640,10641,10642,10643,10644,10645,10646,10647,10648, //10512 -10649,10650,10651,10652,10653,10654,10655,10656,4941,10657,10658,10659,2599,10660,10661,10662, //10528 -10663,10664,10665,10666,3089,10667,10668,10669,10670,10671,10672,10673,10674,10675,10676,10677, //10544 -10678,10679,10680,4942,10681,10682,10683,10684,10685,10686,10687,10688,10689,10690,10691,10692, //10560 -10693,10694,10695,10696,10697,4485,10698,10699,10700,10701,10702,10703,10704,4943,10705,3677, //10576 -10706,10707,10708,10709,10710,10711,10712,4944,10713,10714,10715,10716,10717,10718,10719,10720, //10592 -10721,10722,10723,10724,10725,10726,10727,10728,4945,10729,10730,10731,10732,10733,10734,10735, //10608 -10736,10737,10738,10739,10740,10741,10742,10743,10744,10745,10746,10747,10748,10749,10750,10751, //10624 -10752,10753,10754,10755,10756,10757,10758,10759,10760,10761,4946,10762,10763,10764,10765,10766, //10640 -10767,4947,4948,10768,10769,10770,10771,10772,10773,10774,10775,10776,10777,10778,10779,10780, //10656 -10781,10782,10783,10784,10785,10786,10787,10788,10789,10790,10791,10792,10793,10794,10795,10796, //10672 -10797,10798,10799,10800,10801,10802,10803,10804,10805,10806,10807,10808,10809,10810,10811,10812, //10688 -10813,10814,10815,10816,10817,10818,10819,10820,10821,10822,10823,10824,10825,10826,10827,10828, //10704 -10829,10830,10831,10832,10833,10834,10835,10836,10837,10838,10839,10840,10841,10842,10843,10844, //10720 -10845,10846,10847,10848,10849,10850,10851,10852,10853,10854,10855,10856,10857,10858,10859,10860, //10736 -10861,10862,10863,10864,10865,10866,10867,10868,10869,10870,10871,10872,10873,10874,10875,10876, //10752 -10877,10878,4486,10879,10880,10881,10882,10883,10884,10885,4949,10886,10887,10888,10889,10890, //10768 -10891,10892,10893,10894,10895,10896,10897,10898,10899,10900,10901,10902,10903,10904,10905,10906, //10784 -10907,10908,10909,10910,10911,10912,10913,10914,10915,10916,10917,10918,10919,4487,10920,10921, //10800 -10922,10923,10924,10925,10926,10927,10928,10929,10930,10931,10932,4950,10933,10934,10935,10936, //10816 -10937,10938,10939,10940,10941,10942,10943,10944,10945,10946,10947,10948,10949,4488,10950,10951, //10832 -10952,10953,10954,10955,10956,10957,10958,10959,4190,10960,10961,10962,10963,10964,10965,10966, //10848 -10967,10968,10969,10970,10971,10972,10973,10974,10975,10976,10977,10978,10979,10980,10981,10982, //10864 -10983,10984,10985,10986,10987,10988,10989,10990,10991,10992,10993,10994,10995,10996,10997,10998, //10880 -10999,11000,11001,11002,11003,11004,11005,11006,3960,11007,11008,11009,11010,11011,11012,11013, //10896 -11014,11015,11016,11017,11018,11019,11020,11021,11022,11023,11024,11025,11026,11027,11028,11029, //10912 -11030,11031,11032,4951,11033,11034,11035,11036,11037,11038,11039,11040,11041,11042,11043,11044, //10928 -11045,11046,11047,4489,11048,11049,11050,11051,4952,11052,11053,11054,11055,11056,11057,11058, //10944 -4953,11059,11060,11061,11062,11063,11064,11065,11066,11067,11068,11069,11070,11071,4954,11072, //10960 -11073,11074,11075,11076,11077,11078,11079,11080,11081,11082,11083,11084,11085,11086,11087,11088, //10976 -11089,11090,11091,11092,11093,11094,11095,11096,11097,11098,11099,11100,11101,11102,11103,11104, //10992 -11105,11106,11107,11108,11109,11110,11111,11112,11113,11114,11115,3808,11116,11117,11118,11119, //11008 -11120,11121,11122,11123,11124,11125,11126,11127,11128,11129,11130,11131,11132,11133,11134,4955, //11024 -11135,11136,11137,11138,11139,11140,11141,11142,11143,11144,11145,11146,11147,11148,11149,11150, //11040 -11151,11152,11153,11154,11155,11156,11157,11158,11159,11160,11161,4956,11162,11163,11164,11165, //11056 -11166,11167,11168,11169,11170,11171,11172,11173,11174,11175,11176,11177,11178,11179,11180,4957, //11072 -11181,11182,11183,11184,11185,11186,4958,11187,11188,11189,11190,11191,11192,11193,11194,11195, //11088 -11196,11197,11198,11199,11200,3678,11201,11202,11203,11204,11205,11206,4191,11207,11208,11209, //11104 -11210,11211,11212,11213,11214,11215,11216,11217,11218,11219,11220,11221,11222,11223,11224,11225, //11120 -11226,11227,11228,11229,11230,11231,11232,11233,11234,11235,11236,11237,11238,11239,11240,11241, //11136 -11242,11243,11244,11245,11246,11247,11248,11249,11250,11251,4959,11252,11253,11254,11255,11256, //11152 -11257,11258,11259,11260,11261,11262,11263,11264,11265,11266,11267,11268,11269,11270,11271,11272, //11168 -11273,11274,11275,11276,11277,11278,11279,11280,11281,11282,11283,11284,11285,11286,11287,11288, //11184 -11289,11290,11291,11292,11293,11294,11295,11296,11297,11298,11299,11300,11301,11302,11303,11304, //11200 -11305,11306,11307,11308,11309,11310,11311,11312,11313,11314,3679,11315,11316,11317,11318,4490, //11216 -11319,11320,11321,11322,11323,11324,11325,11326,11327,11328,11329,11330,11331,11332,11333,11334, //11232 -11335,11336,11337,11338,11339,11340,11341,11342,11343,11344,11345,11346,11347,4960,11348,11349, //11248 -11350,11351,11352,11353,11354,11355,11356,11357,11358,11359,11360,11361,11362,11363,11364,11365, //11264 -11366,11367,11368,11369,11370,11371,11372,11373,11374,11375,11376,11377,3961,4961,11378,11379, //11280 -11380,11381,11382,11383,11384,11385,11386,11387,11388,11389,11390,11391,11392,11393,11394,11395, //11296 -11396,11397,4192,11398,11399,11400,11401,11402,11403,11404,11405,11406,11407,11408,11409,11410, //11312 -11411,4962,11412,11413,11414,11415,11416,11417,11418,11419,11420,11421,11422,11423,11424,11425, //11328 -11426,11427,11428,11429,11430,11431,11432,11433,11434,11435,11436,11437,11438,11439,11440,11441, //11344 -11442,11443,11444,11445,11446,11447,11448,11449,11450,11451,11452,11453,11454,11455,11456,11457, //11360 -11458,11459,11460,11461,11462,11463,11464,11465,11466,11467,11468,11469,4963,11470,11471,4491, //11376 -11472,11473,11474,11475,4964,11476,11477,11478,11479,11480,11481,11482,11483,11484,11485,11486, //11392 -11487,11488,11489,11490,11491,11492,4965,11493,11494,11495,11496,11497,11498,11499,11500,11501, //11408 -11502,11503,11504,11505,11506,11507,11508,11509,11510,11511,11512,11513,11514,11515,11516,11517, //11424 -11518,11519,11520,11521,11522,11523,11524,11525,11526,11527,11528,11529,3962,11530,11531,11532, //11440 -11533,11534,11535,11536,11537,11538,11539,11540,11541,11542,11543,11544,11545,11546,11547,11548, //11456 -11549,11550,11551,11552,11553,11554,11555,11556,11557,11558,11559,11560,11561,11562,11563,11564, //11472 -4193,4194,11565,11566,11567,11568,11569,11570,11571,11572,11573,11574,11575,11576,11577,11578, //11488 -11579,11580,11581,11582,11583,11584,11585,11586,11587,11588,11589,11590,11591,4966,4195,11592, //11504 -11593,11594,11595,11596,11597,11598,11599,11600,11601,11602,11603,11604,3090,11605,11606,11607, //11520 -11608,11609,11610,4967,11611,11612,11613,11614,11615,11616,11617,11618,11619,11620,11621,11622, //11536 -11623,11624,11625,11626,11627,11628,11629,11630,11631,11632,11633,11634,11635,11636,11637,11638, //11552 -11639,11640,11641,11642,11643,11644,11645,11646,11647,11648,11649,11650,11651,11652,11653,11654, //11568 -11655,11656,11657,11658,11659,11660,11661,11662,11663,11664,11665,11666,11667,11668,11669,11670, //11584 -11671,11672,11673,11674,4968,11675,11676,11677,11678,11679,11680,11681,11682,11683,11684,11685, //11600 -11686,11687,11688,11689,11690,11691,11692,11693,3809,11694,11695,11696,11697,11698,11699,11700, //11616 -11701,11702,11703,11704,11705,11706,11707,11708,11709,11710,11711,11712,11713,11714,11715,11716, //11632 -11717,11718,3553,11719,11720,11721,11722,11723,11724,11725,11726,11727,11728,11729,11730,4969, //11648 -11731,11732,11733,11734,11735,11736,11737,11738,11739,11740,4492,11741,11742,11743,11744,11745, //11664 -11746,11747,11748,11749,11750,11751,11752,4970,11753,11754,11755,11756,11757,11758,11759,11760, //11680 -11761,11762,11763,11764,11765,11766,11767,11768,11769,11770,11771,11772,11773,11774,11775,11776, //11696 -11777,11778,11779,11780,11781,11782,11783,11784,11785,11786,11787,11788,11789,11790,4971,11791, //11712 -11792,11793,11794,11795,11796,11797,4972,11798,11799,11800,11801,11802,11803,11804,11805,11806, //11728 -11807,11808,11809,11810,4973,11811,11812,11813,11814,11815,11816,11817,11818,11819,11820,11821, //11744 -11822,11823,11824,11825,11826,11827,11828,11829,11830,11831,11832,11833,11834,3680,3810,11835, //11760 -11836,4974,11837,11838,11839,11840,11841,11842,11843,11844,11845,11846,11847,11848,11849,11850, //11776 -11851,11852,11853,11854,11855,11856,11857,11858,11859,11860,11861,11862,11863,11864,11865,11866, //11792 -11867,11868,11869,11870,11871,11872,11873,11874,11875,11876,11877,11878,11879,11880,11881,11882, //11808 -11883,11884,4493,11885,11886,11887,11888,11889,11890,11891,11892,11893,11894,11895,11896,11897, //11824 -11898,11899,11900,11901,11902,11903,11904,11905,11906,11907,11908,11909,11910,11911,11912,11913, //11840 -11914,11915,4975,11916,11917,11918,11919,11920,11921,11922,11923,11924,11925,11926,11927,11928, //11856 -11929,11930,11931,11932,11933,11934,11935,11936,11937,11938,11939,11940,11941,11942,11943,11944, //11872 -11945,11946,11947,11948,11949,4976,11950,11951,11952,11953,11954,11955,11956,11957,11958,11959, //11888 -11960,11961,11962,11963,11964,11965,11966,11967,11968,11969,11970,11971,11972,11973,11974,11975, //11904 -11976,11977,11978,11979,11980,11981,11982,11983,11984,11985,11986,11987,4196,11988,11989,11990, //11920 -11991,11992,4977,11993,11994,11995,11996,11997,11998,11999,12000,12001,12002,12003,12004,12005, //11936 -12006,12007,12008,12009,12010,12011,12012,12013,12014,12015,12016,12017,12018,12019,12020,12021, //11952 -12022,12023,12024,12025,12026,12027,12028,12029,12030,12031,12032,12033,12034,12035,12036,12037, //11968 -12038,12039,12040,12041,12042,12043,12044,12045,12046,12047,12048,12049,12050,12051,12052,12053, //11984 -12054,12055,12056,12057,12058,12059,12060,12061,4978,12062,12063,12064,12065,12066,12067,12068, //12000 -12069,12070,12071,12072,12073,12074,12075,12076,12077,12078,12079,12080,12081,12082,12083,12084, //12016 -12085,12086,12087,12088,12089,12090,12091,12092,12093,12094,12095,12096,12097,12098,12099,12100, //12032 -12101,12102,12103,12104,12105,12106,12107,12108,12109,12110,12111,12112,12113,12114,12115,12116, //12048 -12117,12118,12119,12120,12121,12122,12123,4979,12124,12125,12126,12127,12128,4197,12129,12130, //12064 -12131,12132,12133,12134,12135,12136,12137,12138,12139,12140,12141,12142,12143,12144,12145,12146, //12080 -12147,12148,12149,12150,12151,12152,12153,12154,4980,12155,12156,12157,12158,12159,12160,4494, //12096 -12161,12162,12163,12164,3811,12165,12166,12167,12168,12169,4495,12170,12171,4496,12172,12173, //12112 -12174,12175,12176,3812,12177,12178,12179,12180,12181,12182,12183,12184,12185,12186,12187,12188, //12128 -12189,12190,12191,12192,12193,12194,12195,12196,12197,12198,12199,12200,12201,12202,12203,12204, //12144 -12205,12206,12207,12208,12209,12210,12211,12212,12213,12214,12215,12216,12217,12218,12219,12220, //12160 -12221,4981,12222,12223,12224,12225,12226,12227,12228,12229,12230,12231,12232,12233,12234,12235, //12176 -4982,12236,12237,12238,12239,12240,12241,12242,12243,12244,12245,4983,12246,12247,12248,12249, //12192 -4984,12250,12251,12252,12253,12254,12255,12256,12257,12258,12259,12260,12261,12262,12263,12264, //12208 -4985,12265,4497,12266,12267,12268,12269,12270,12271,12272,12273,12274,12275,12276,12277,12278, //12224 -12279,12280,12281,12282,12283,12284,12285,12286,12287,4986,12288,12289,12290,12291,12292,12293, //12240 -12294,12295,12296,2473,12297,12298,12299,12300,12301,12302,12303,12304,12305,12306,12307,12308, //12256 -12309,12310,12311,12312,12313,12314,12315,12316,12317,12318,12319,3963,12320,12321,12322,12323, //12272 -12324,12325,12326,12327,12328,12329,12330,12331,12332,4987,12333,12334,12335,12336,12337,12338, //12288 -12339,12340,12341,12342,12343,12344,12345,12346,12347,12348,12349,12350,12351,12352,12353,12354, //12304 -12355,12356,12357,12358,12359,3964,12360,12361,12362,12363,12364,12365,12366,12367,12368,12369, //12320 -12370,3965,12371,12372,12373,12374,12375,12376,12377,12378,12379,12380,12381,12382,12383,12384, //12336 -12385,12386,12387,12388,12389,12390,12391,12392,12393,12394,12395,12396,12397,12398,12399,12400, //12352 -12401,12402,12403,12404,12405,12406,12407,12408,4988,12409,12410,12411,12412,12413,12414,12415, //12368 -12416,12417,12418,12419,12420,12421,12422,12423,12424,12425,12426,12427,12428,12429,12430,12431, //12384 -12432,12433,12434,12435,12436,12437,12438,3554,12439,12440,12441,12442,12443,12444,12445,12446, //12400 -12447,12448,12449,12450,12451,12452,12453,12454,12455,12456,12457,12458,12459,12460,12461,12462, //12416 -12463,12464,4989,12465,12466,12467,12468,12469,12470,12471,12472,12473,12474,12475,12476,12477, //12432 -12478,12479,12480,4990,12481,12482,12483,12484,12485,12486,12487,12488,12489,4498,12490,12491, //12448 -12492,12493,12494,12495,12496,12497,12498,12499,12500,12501,12502,12503,12504,12505,12506,12507, //12464 -12508,12509,12510,12511,12512,12513,12514,12515,12516,12517,12518,12519,12520,12521,12522,12523, //12480 -12524,12525,12526,12527,12528,12529,12530,12531,12532,12533,12534,12535,12536,12537,12538,12539, //12496 -12540,12541,12542,12543,12544,12545,12546,12547,12548,12549,12550,12551,4991,12552,12553,12554, //12512 -12555,12556,12557,12558,12559,12560,12561,12562,12563,12564,12565,12566,12567,12568,12569,12570, //12528 -12571,12572,12573,12574,12575,12576,12577,12578,3036,12579,12580,12581,12582,12583,3966,12584, //12544 -12585,12586,12587,12588,12589,12590,12591,12592,12593,12594,12595,12596,12597,12598,12599,12600, //12560 -12601,12602,12603,12604,12605,12606,12607,12608,12609,12610,12611,12612,12613,12614,12615,12616, //12576 -12617,12618,12619,12620,12621,12622,12623,12624,12625,12626,12627,12628,12629,12630,12631,12632, //12592 -12633,12634,12635,12636,12637,12638,12639,12640,12641,12642,12643,12644,12645,12646,4499,12647, //12608 -12648,12649,12650,12651,12652,12653,12654,12655,12656,12657,12658,12659,12660,12661,12662,12663, //12624 -12664,12665,12666,12667,12668,12669,12670,12671,12672,12673,12674,12675,12676,12677,12678,12679, //12640 -12680,12681,12682,12683,12684,12685,12686,12687,12688,12689,12690,12691,12692,12693,12694,12695, //12656 -12696,12697,12698,4992,12699,12700,12701,12702,12703,12704,12705,12706,12707,12708,12709,12710, //12672 -12711,12712,12713,12714,12715,12716,12717,12718,12719,12720,12721,12722,12723,12724,12725,12726, //12688 -12727,12728,12729,12730,12731,12732,12733,12734,12735,12736,12737,12738,12739,12740,12741,12742, //12704 -12743,12744,12745,12746,12747,12748,12749,12750,12751,12752,12753,12754,12755,12756,12757,12758, //12720 -12759,12760,12761,12762,12763,12764,12765,12766,12767,12768,12769,12770,12771,12772,12773,12774, //12736 -12775,12776,12777,12778,4993,2175,12779,12780,12781,12782,12783,12784,12785,12786,4500,12787, //12752 -12788,12789,12790,12791,12792,12793,12794,12795,12796,12797,12798,12799,12800,12801,12802,12803, //12768 -12804,12805,12806,12807,12808,12809,12810,12811,12812,12813,12814,12815,12816,12817,12818,12819, //12784 -12820,12821,12822,12823,12824,12825,12826,4198,3967,12827,12828,12829,12830,12831,12832,12833, //12800 -12834,12835,12836,12837,12838,12839,12840,12841,12842,12843,12844,12845,12846,12847,12848,12849, //12816 -12850,12851,12852,12853,12854,12855,12856,12857,12858,12859,12860,12861,4199,12862,12863,12864, //12832 -12865,12866,12867,12868,12869,12870,12871,12872,12873,12874,12875,12876,12877,12878,12879,12880, //12848 -12881,12882,12883,12884,12885,12886,12887,4501,12888,12889,12890,12891,12892,12893,12894,12895, //12864 -12896,12897,12898,12899,12900,12901,12902,12903,12904,12905,12906,12907,12908,12909,12910,12911, //12880 -12912,4994,12913,12914,12915,12916,12917,12918,12919,12920,12921,12922,12923,12924,12925,12926, //12896 -12927,12928,12929,12930,12931,12932,12933,12934,12935,12936,12937,12938,12939,12940,12941,12942, //12912 -12943,12944,12945,12946,12947,12948,12949,12950,12951,12952,12953,12954,12955,12956,1772,12957, //12928 -12958,12959,12960,12961,12962,12963,12964,12965,12966,12967,12968,12969,12970,12971,12972,12973, //12944 -12974,12975,12976,12977,12978,12979,12980,12981,12982,12983,12984,12985,12986,12987,12988,12989, //12960 -12990,12991,12992,12993,12994,12995,12996,12997,4502,12998,4503,12999,13000,13001,13002,13003, //12976 -4504,13004,13005,13006,13007,13008,13009,13010,13011,13012,13013,13014,13015,13016,13017,13018, //12992 -13019,13020,13021,13022,13023,13024,13025,13026,13027,13028,13029,3449,13030,13031,13032,13033, //13008 -13034,13035,13036,13037,13038,13039,13040,13041,13042,13043,13044,13045,13046,13047,13048,13049, //13024 -13050,13051,13052,13053,13054,13055,13056,13057,13058,13059,13060,13061,13062,13063,13064,13065, //13040 -13066,13067,13068,13069,13070,13071,13072,13073,13074,13075,13076,13077,13078,13079,13080,13081, //13056 -13082,13083,13084,13085,13086,13087,13088,13089,13090,13091,13092,13093,13094,13095,13096,13097, //13072 -13098,13099,13100,13101,13102,13103,13104,13105,13106,13107,13108,13109,13110,13111,13112,13113, //13088 -13114,13115,13116,13117,13118,3968,13119,4995,13120,13121,13122,13123,13124,13125,13126,13127, //13104 -4505,13128,13129,13130,13131,13132,13133,13134,4996,4506,13135,13136,13137,13138,13139,4997, //13120 -13140,13141,13142,13143,13144,13145,13146,13147,13148,13149,13150,13151,13152,13153,13154,13155, //13136 -13156,13157,13158,13159,4998,13160,13161,13162,13163,13164,13165,13166,13167,13168,13169,13170, //13152 -13171,13172,13173,13174,13175,13176,4999,13177,13178,13179,13180,13181,13182,13183,13184,13185, //13168 -13186,13187,13188,13189,13190,13191,13192,13193,13194,13195,13196,13197,13198,13199,13200,13201, //13184 -13202,13203,13204,13205,13206,5000,13207,13208,13209,13210,13211,13212,13213,13214,13215,13216, //13200 -13217,13218,13219,13220,13221,13222,13223,13224,13225,13226,13227,4200,5001,13228,13229,13230, //13216 -13231,13232,13233,13234,13235,13236,13237,13238,13239,13240,3969,13241,13242,13243,13244,3970, //13232 -13245,13246,13247,13248,13249,13250,13251,13252,13253,13254,13255,13256,13257,13258,13259,13260, //13248 -13261,13262,13263,13264,13265,13266,13267,13268,3450,13269,13270,13271,13272,13273,13274,13275, //13264 -13276,5002,13277,13278,13279,13280,13281,13282,13283,13284,13285,13286,13287,13288,13289,13290, //13280 -13291,13292,13293,13294,13295,13296,13297,13298,13299,13300,13301,13302,3813,13303,13304,13305, //13296 -13306,13307,13308,13309,13310,13311,13312,13313,13314,13315,13316,13317,13318,13319,13320,13321, //13312 -13322,13323,13324,13325,13326,13327,13328,4507,13329,13330,13331,13332,13333,13334,13335,13336, //13328 -13337,13338,13339,13340,13341,5003,13342,13343,13344,13345,13346,13347,13348,13349,13350,13351, //13344 -13352,13353,13354,13355,13356,13357,13358,13359,13360,13361,13362,13363,13364,13365,13366,13367, //13360 -5004,13368,13369,13370,13371,13372,13373,13374,13375,13376,13377,13378,13379,13380,13381,13382, //13376 -13383,13384,13385,13386,13387,13388,13389,13390,13391,13392,13393,13394,13395,13396,13397,13398, //13392 -13399,13400,13401,13402,13403,13404,13405,13406,13407,13408,13409,13410,13411,13412,13413,13414, //13408 -13415,13416,13417,13418,13419,13420,13421,13422,13423,13424,13425,13426,13427,13428,13429,13430, //13424 -13431,13432,4508,13433,13434,13435,4201,13436,13437,13438,13439,13440,13441,13442,13443,13444, //13440 -13445,13446,13447,13448,13449,13450,13451,13452,13453,13454,13455,13456,13457,5005,13458,13459, //13456 -13460,13461,13462,13463,13464,13465,13466,13467,13468,13469,13470,4509,13471,13472,13473,13474, //13472 -13475,13476,13477,13478,13479,13480,13481,13482,13483,13484,13485,13486,13487,13488,13489,13490, //13488 -13491,13492,13493,13494,13495,13496,13497,13498,13499,13500,13501,13502,13503,13504,13505,13506, //13504 -13507,13508,13509,13510,13511,13512,13513,13514,13515,13516,13517,13518,13519,13520,13521,13522, //13520 -13523,13524,13525,13526,13527,13528,13529,13530,13531,13532,13533,13534,13535,13536,13537,13538, //13536 -13539,13540,13541,13542,13543,13544,13545,13546,13547,13548,13549,13550,13551,13552,13553,13554, //13552 -13555,13556,13557,13558,13559,13560,13561,13562,13563,13564,13565,13566,13567,13568,13569,13570, //13568 -13571,13572,13573,13574,13575,13576,13577,13578,13579,13580,13581,13582,13583,13584,13585,13586, //13584 -13587,13588,13589,13590,13591,13592,13593,13594,13595,13596,13597,13598,13599,13600,13601,13602, //13600 -13603,13604,13605,13606,13607,13608,13609,13610,13611,13612,13613,13614,13615,13616,13617,13618, //13616 -13619,13620,13621,13622,13623,13624,13625,13626,13627,13628,13629,13630,13631,13632,13633,13634, //13632 -13635,13636,13637,13638,13639,13640,13641,13642,5006,13643,13644,13645,13646,13647,13648,13649, //13648 -13650,13651,5007,13652,13653,13654,13655,13656,13657,13658,13659,13660,13661,13662,13663,13664, //13664 -13665,13666,13667,13668,13669,13670,13671,13672,13673,13674,13675,13676,13677,13678,13679,13680, //13680 -13681,13682,13683,13684,13685,13686,13687,13688,13689,13690,13691,13692,13693,13694,13695,13696, //13696 -13697,13698,13699,13700,13701,13702,13703,13704,13705,13706,13707,13708,13709,13710,13711,13712, //13712 -13713,13714,13715,13716,13717,13718,13719,13720,13721,13722,13723,13724,13725,13726,13727,13728, //13728 -13729,13730,13731,13732,13733,13734,13735,13736,13737,13738,13739,13740,13741,13742,13743,13744, //13744 -13745,13746,13747,13748,13749,13750,13751,13752,13753,13754,13755,13756,13757,13758,13759,13760, //13760 -13761,13762,13763,13764,13765,13766,13767,13768,13769,13770,13771,13772,13773,13774,3273,13775, //13776 -13776,13777,13778,13779,13780,13781,13782,13783,13784,13785,13786,13787,13788,13789,13790,13791, //13792 -13792,13793,13794,13795,13796,13797,13798,13799,13800,13801,13802,13803,13804,13805,13806,13807, //13808 -13808,13809,13810,13811,13812,13813,13814,13815,13816,13817,13818,13819,13820,13821,13822,13823, //13824 -13824,13825,13826,13827,13828,13829,13830,13831,13832,13833,13834,13835,13836,13837,13838,13839, //13840 -13840,13841,13842,13843,13844,13845,13846,13847,13848,13849,13850,13851,13852,13853,13854,13855, //13856 -13856,13857,13858,13859,13860,13861,13862,13863,13864,13865,13866,13867,13868,13869,13870,13871, //13872 -13872,13873,13874,13875,13876,13877,13878,13879,13880,13881,13882,13883,13884,13885,13886,13887, //13888 -13888,13889,13890,13891,13892,13893,13894,13895,13896,13897,13898,13899,13900,13901,13902,13903, //13904 -13904,13905,13906,13907,13908,13909,13910,13911,13912,13913,13914,13915,13916,13917,13918,13919, //13920 -13920,13921,13922,13923,13924,13925,13926,13927,13928,13929,13930,13931,13932,13933,13934,13935, //13936 -13936,13937,13938,13939,13940,13941,13942,13943,13944,13945,13946,13947,13948,13949,13950,13951, //13952 -13952,13953,13954,13955,13956,13957,13958,13959,13960,13961,13962,13963,13964,13965,13966,13967, //13968 -13968,13969,13970,13971,13972, //13973 -****************************************************************************************/ -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/CharDistribution.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/CharDistribution.cpp deleted file mode 100644 index 488d9bc..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/CharDistribution.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "CharDistribution.h" - -#include "JISFreq.tab" -#include "Big5Freq.tab" -#include "EUCKRFreq.tab" -#include "EUCTWFreq.tab" -#include "GB2312Freq.tab" - -#define SURE_YES 0.99f -#define SURE_NO 0.01f - -//return confidence base on received data -float CharDistributionAnalysis::GetConfidence(void) -{ - //if we didn't receive any character in our consideration range, or the - // number of frequent characters is below the minimum threshold, return - // negative answer - if (mTotalChars <= 0 || mFreqChars <= mDataThreshold) - return SURE_NO; - - if (mTotalChars != mFreqChars) { - float r = mFreqChars / ((mTotalChars - mFreqChars) * mTypicalDistributionRatio); - - if (r < SURE_YES) - return r; - } - //normalize confidence, (we don't want to be 100% sure) - return SURE_YES; -} - -EUCTWDistributionAnalysis::EUCTWDistributionAnalysis() -{ - mCharToFreqOrder = EUCTWCharToFreqOrder; - mTableSize = EUCTW_TABLE_SIZE; - mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO; -} - -EUCKRDistributionAnalysis::EUCKRDistributionAnalysis() -{ - mCharToFreqOrder = EUCKRCharToFreqOrder; - mTableSize = EUCKR_TABLE_SIZE; - mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO; -} - -GB2312DistributionAnalysis::GB2312DistributionAnalysis() -{ - mCharToFreqOrder = GB2312CharToFreqOrder; - mTableSize = GB2312_TABLE_SIZE; - mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO; -} - -Big5DistributionAnalysis::Big5DistributionAnalysis() -{ - mCharToFreqOrder = Big5CharToFreqOrder; - mTableSize = BIG5_TABLE_SIZE; - mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO; -} - -SJISDistributionAnalysis::SJISDistributionAnalysis() -{ - mCharToFreqOrder = JISCharToFreqOrder; - mTableSize = JIS_TABLE_SIZE; - mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO; -} - -EUCJPDistributionAnalysis::EUCJPDistributionAnalysis() -{ - mCharToFreqOrder = JISCharToFreqOrder; - mTableSize = JIS_TABLE_SIZE; - mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO; -} - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/CharDistribution.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/CharDistribution.h deleted file mode 100644 index 453c2de..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/CharDistribution.h +++ /dev/null @@ -1,242 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef CharDistribution_h__ -#define CharDistribution_h__ - -#include "nscore.h" - -#define ENOUGH_DATA_THRESHOLD 1024 - -#define MINIMUM_DATA_THRESHOLD 4 - -class CharDistributionAnalysis -{ -public: - CharDistributionAnalysis() {Reset(PR_FALSE);} - - //feed a block of data and do distribution analysis - void HandleData(const char* aBuf, PRUint32 aLen) {} - - //Feed a character with known length - void HandleOneChar(const char* aStr, PRUint32 aCharLen) - { - PRInt32 order; - - //we only care about 2-bytes character in our distribution analysis - order = (aCharLen == 2) ? GetOrder(aStr) : -1; - - if (order >= 0) - { - mTotalChars++; - //order is valid - if ((PRUint32)order < mTableSize) - { - if (512 > mCharToFreqOrder[order]) - mFreqChars++; - } - } - } - - //return confidence base on existing data - float GetConfidence(void); - - //Reset analyser, clear any state - void Reset(PRBool aIsPreferredLanguage) - { - mDone = PR_FALSE; - mTotalChars = 0; - mFreqChars = 0; - mDataThreshold = aIsPreferredLanguage ? 0 : MINIMUM_DATA_THRESHOLD; - } - - //This function is for future extension. Caller can use this function to control - //analyser's behavior - void SetOpion(){} - - //It is not necessary to receive all data to draw conclusion. For charset detection, - // certain amount of data is enough - PRBool GotEnoughData() {return mTotalChars > ENOUGH_DATA_THRESHOLD;} - -protected: - //we do not handle character base on its original encoding string, but - //convert this encoding string to a number, here called order. - //This allow multiple encoding of a language to share one frequency table - virtual PRInt32 GetOrder(const char* str) {return -1;} - - //If this flag is set to PR_TRUE, detection is done and conclusion has been made - PRBool mDone; - - //The number of characters whose frequency order is less than 512 - PRUint32 mFreqChars; - - //Total character encounted. - PRUint32 mTotalChars; - - //Number of hi-byte characters needed to trigger detection - PRUint32 mDataThreshold; - - //Mapping table to get frequency order from char order (get from GetOrder()) - const PRInt16 *mCharToFreqOrder; - - //Size of above table - PRUint32 mTableSize; - - //This is a constant value varies from language to language, it is used in - //calculating confidence. See my paper for further detail. - float mTypicalDistributionRatio; -}; - - -class EUCTWDistributionAnalysis: public CharDistributionAnalysis -{ -public: - EUCTWDistributionAnalysis(); -protected: - - //for euc-TW encoding, we are interested - // first byte range: 0xc4 -- 0xfe - // second byte range: 0xa1 -- 0xfe - //no validation needed here. State machine has done that - PRInt32 GetOrder(const char* str) - { if ((unsigned char)*str >= (unsigned char)0xc4) - return 94*((unsigned char)str[0]-(unsigned char)0xc4) + (unsigned char)str[1] - (unsigned char)0xa1; - else - return -1; - } -}; - - -class EUCKRDistributionAnalysis : public CharDistributionAnalysis -{ -public: - EUCKRDistributionAnalysis(); -protected: - //for euc-KR encoding, we are interested - // first byte range: 0xb0 -- 0xfe - // second byte range: 0xa1 -- 0xfe - //no validation needed here. State machine has done that - PRInt32 GetOrder(const char* str) - { if ((unsigned char)*str >= (unsigned char)0xb0) - return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1; - else - return -1; - } -}; - -class GB2312DistributionAnalysis : public CharDistributionAnalysis -{ -public: - GB2312DistributionAnalysis(); -protected: - //for GB2312 encoding, we are interested - // first byte range: 0xb0 -- 0xfe - // second byte range: 0xa1 -- 0xfe - //no validation needed here. State machine has done that - PRInt32 GetOrder(const char* str) - { if ((unsigned char)*str >= (unsigned char)0xb0 && (unsigned char)str[1] >= (unsigned char)0xa1) - return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1; - else - return -1; - } -}; - - -class Big5DistributionAnalysis : public CharDistributionAnalysis -{ -public: - Big5DistributionAnalysis(); -protected: - //for big5 encoding, we are interested - // first byte range: 0xa4 -- 0xfe - // second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe - //no validation needed here. State machine has done that - PRInt32 GetOrder(const char* str) - { if ((unsigned char)*str >= (unsigned char)0xa4) - if ((unsigned char)str[1] >= (unsigned char)0xa1) - return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0xa1 +63; - else - return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0x40; - else - return -1; - } -}; - -class SJISDistributionAnalysis : public CharDistributionAnalysis -{ -public: - SJISDistributionAnalysis(); -protected: - //for sjis encoding, we are interested - // first byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe - // second byte range: 0x40 -- 0x7e, 0x81 -- oxfe - //no validation needed here. State machine has done that - PRInt32 GetOrder(const char* str) - { - PRInt32 order; - if ((unsigned char)*str >= (unsigned char)0x81 && (unsigned char)*str <= (unsigned char)0x9f) - order = 188 * ((unsigned char)str[0]-(unsigned char)0x81); - else if ((unsigned char)*str >= (unsigned char)0xe0 && (unsigned char)*str <= (unsigned char)0xef) - order = 188 * ((unsigned char)str[0]-(unsigned char)0xe0 + 31); - else - return -1; - order += (unsigned char)*(str+1) - 0x40; - if ((unsigned char)str[1] > (unsigned char)0x7f) - order--; - return order; - } -}; - -class EUCJPDistributionAnalysis : public CharDistributionAnalysis -{ -public: - EUCJPDistributionAnalysis(); -protected: - //for euc-JP encoding, we are interested - // first byte range: 0xa0 -- 0xfe - // second byte range: 0xa1 -- 0xfe - //no validation needed here. State machine has done that - PRInt32 GetOrder(const char* str) - { if ((unsigned char)*str >= (unsigned char)0xa0) - return 94*((unsigned char)str[0]-(unsigned char)0xa1) + (unsigned char)str[1] - (unsigned char)0xa1; - else - return -1; - } -}; - -#endif //CharDistribution_h__ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/EUCKRFreq.tab b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/EUCKRFreq.tab deleted file mode 100644 index d2c5047..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/EUCKRFreq.tab +++ /dev/null @@ -1,614 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -//Sampling from about 20M text materials include literature and computer technology - -/****************************************************************************** - * 128 --> 0.79 - * 256 --> 0.92 - * 512 --> 0.986 - * 1024 --> 0.99944 - * 2048 --> 0.99999 - * - * Idea Distribution Ratio = 0.98653 / (1-0.98653) = 73.24 - * Random Distribution Ration = 512 / (2350-512) = 0.279. - * - * Typical Distribution Ratio - *****************************************************************************/ - -#define EUCKR_TYPICAL_DISTRIBUTION_RATIO (float) 6.0 - -#define EUCKR_TABLE_SIZE 2352 - -//Char to FreqOrder table , -static const PRInt16 EUCKRCharToFreqOrder[] = -{ - 13, 130, 120,1396, 481,1719,1720, 328, 609, 212,1721, 707, 400, 299,1722, 87, -1397,1723, 104, 536,1117,1203,1724,1267, 685,1268, 508,1725,1726,1727,1728,1398, -1399,1729,1730,1731, 141, 621, 326,1057, 368,1732, 267, 488, 20,1733,1269,1734, - 945,1400,1735, 47, 904,1270,1736,1737, 773, 248,1738, 409, 313, 786, 429,1739, - 116, 987, 813,1401, 683, 75,1204, 145,1740,1741,1742,1743, 16, 847, 667, 622, - 708,1744,1745,1746, 966, 787, 304, 129,1747, 60, 820, 123, 676,1748,1749,1750, -1751, 617,1752, 626,1753,1754,1755,1756, 653,1757,1758,1759,1760,1761,1762, 856, - 344,1763,1764,1765,1766, 89, 401, 418, 806, 905, 848,1767,1768,1769, 946,1205, - 709,1770,1118,1771, 241,1772,1773,1774,1271,1775, 569,1776, 999,1777,1778,1779, -1780, 337, 751,1058, 28, 628, 254,1781, 177, 906, 270, 349, 891,1079,1782, 19, -1783, 379,1784, 315,1785, 629, 754,1402, 559,1786, 636, 203,1206,1787, 710, 567, -1788, 935, 814,1789,1790,1207, 766, 528,1791,1792,1208,1793,1794,1795,1796,1797, -1403,1798,1799, 533,1059,1404,1405,1156,1406, 936, 884,1080,1800, 351,1801,1802, -1803,1804,1805, 801,1806,1807,1808,1119,1809,1157, 714, 474,1407,1810, 298, 899, - 885,1811,1120, 802,1158,1812, 892,1813,1814,1408, 659,1815,1816,1121,1817,1818, -1819,1820,1821,1822, 319,1823, 594, 545,1824, 815, 937,1209,1825,1826, 573,1409, -1022,1827,1210,1828,1829,1830,1831,1832,1833, 556, 722, 807,1122,1060,1834, 697, -1835, 900, 557, 715,1836,1410, 540,1411, 752,1159, 294, 597,1211, 976, 803, 770, -1412,1837,1838, 39, 794,1413, 358,1839, 371, 925,1840, 453, 661, 788, 531, 723, - 544,1023,1081, 869, 91,1841, 392, 430, 790, 602,1414, 677,1082, 457,1415,1416, -1842,1843, 475, 327,1024,1417, 795, 121,1844, 733, 403,1418,1845,1846,1847, 300, - 119, 711,1212, 627,1848,1272, 207,1849,1850, 796,1213, 382,1851, 519,1852,1083, - 893,1853,1854,1855, 367, 809, 487, 671,1856, 663,1857,1858, 956, 471, 306, 857, -1859,1860,1160,1084,1861,1862,1863,1864,1865,1061,1866,1867,1868,1869,1870,1871, - 282, 96, 574,1872, 502,1085,1873,1214,1874, 907,1875,1876, 827, 977,1419,1420, -1421, 268,1877,1422,1878,1879,1880, 308,1881, 2, 537,1882,1883,1215,1884,1885, - 127, 791,1886,1273,1423,1887, 34, 336, 404, 643,1888, 571, 654, 894, 840,1889, - 0, 886,1274, 122, 575, 260, 908, 938,1890,1275, 410, 316,1891,1892, 100,1893, -1894,1123, 48,1161,1124,1025,1895, 633, 901,1276,1896,1897, 115, 816,1898, 317, -1899, 694,1900, 909, 734,1424, 572, 866,1425, 691, 85, 524,1010, 543, 394, 841, -1901,1902,1903,1026,1904,1905,1906,1907,1908,1909, 30, 451, 651, 988, 310,1910, -1911,1426, 810,1216, 93,1912,1913,1277,1217,1914, 858, 759, 45, 58, 181, 610, - 269,1915,1916, 131,1062, 551, 443,1000, 821,1427, 957, 895,1086,1917,1918, 375, -1919, 359,1920, 687,1921, 822,1922, 293,1923,1924, 40, 662, 118, 692, 29, 939, - 887, 640, 482, 174,1925, 69,1162, 728,1428, 910,1926,1278,1218,1279, 386, 870, - 217, 854,1163, 823,1927,1928,1929,1930, 834,1931, 78,1932, 859,1933,1063,1934, -1935,1936,1937, 438,1164, 208, 595,1938,1939,1940,1941,1219,1125,1942, 280, 888, -1429,1430,1220,1431,1943,1944,1945,1946,1947,1280, 150, 510,1432,1948,1949,1950, -1951,1952,1953,1954,1011,1087,1955,1433,1043,1956, 881,1957, 614, 958,1064,1065, -1221,1958, 638,1001, 860, 967, 896,1434, 989, 492, 553,1281,1165,1959,1282,1002, -1283,1222,1960,1961,1962,1963, 36, 383, 228, 753, 247, 454,1964, 876, 678,1965, -1966,1284, 126, 464, 490, 835, 136, 672, 529, 940,1088,1435, 473,1967,1968, 467, - 50, 390, 227, 587, 279, 378, 598, 792, 968, 240, 151, 160, 849, 882,1126,1285, - 639,1044, 133, 140, 288, 360, 811, 563,1027, 561, 142, 523,1969,1970,1971, 7, - 103, 296, 439, 407, 506, 634, 990,1972,1973,1974,1975, 645,1976,1977,1978,1979, -1980,1981, 236,1982,1436,1983,1984,1089, 192, 828, 618, 518,1166, 333,1127,1985, - 818,1223,1986,1987,1988,1989,1990,1991,1992,1993, 342,1128,1286, 746, 842,1994, -1995, 560, 223,1287, 98, 8, 189, 650, 978,1288,1996,1437,1997, 17, 345, 250, - 423, 277, 234, 512, 226, 97, 289, 42, 167,1998, 201,1999,2000, 843, 836, 824, - 532, 338, 783,1090, 182, 576, 436,1438,1439, 527, 500,2001, 947, 889,2002,2003, -2004,2005, 262, 600, 314, 447,2006, 547,2007, 693, 738,1129,2008, 71,1440, 745, - 619, 688,2009, 829,2010,2011, 147,2012, 33, 948,2013,2014, 74, 224,2015, 61, - 191, 918, 399, 637,2016,1028,1130, 257, 902,2017,2018,2019,2020,2021,2022,2023, -2024,2025,2026, 837,2027,2028,2029,2030, 179, 874, 591, 52, 724, 246,2031,2032, -2033,2034,1167, 969,2035,1289, 630, 605, 911,1091,1168,2036,2037,2038,1441, 912, -2039, 623,2040,2041, 253,1169,1290,2042,1442, 146, 620, 611, 577, 433,2043,1224, - 719,1170, 959, 440, 437, 534, 84, 388, 480,1131, 159, 220, 198, 679,2044,1012, - 819,1066,1443, 113,1225, 194, 318,1003,1029,2045,2046,2047,2048,1067,2049,2050, -2051,2052,2053, 59, 913, 112,2054, 632,2055, 455, 144, 739,1291,2056, 273, 681, - 499,2057, 448,2058,2059, 760,2060,2061, 970, 384, 169, 245,1132,2062,2063, 414, -1444,2064,2065, 41, 235,2066, 157, 252, 877, 568, 919, 789, 580,2067, 725,2068, -2069,1292,2070,2071,1445,2072,1446,2073,2074, 55, 588, 66,1447, 271,1092,2075, -1226,2076, 960,1013, 372,2077,2078,2079,2080,2081,1293,2082,2083,2084,2085, 850, -2086,2087,2088,2089,2090, 186,2091,1068, 180,2092,2093,2094, 109,1227, 522, 606, -2095, 867,1448,1093, 991,1171, 926, 353,1133,2096, 581,2097,2098,2099,1294,1449, -1450,2100, 596,1172,1014,1228,2101,1451,1295,1173,1229,2102,2103,1296,1134,1452, - 949,1135,2104,2105,1094,1453,1454,1455,2106,1095,2107,2108,2109,2110,2111,2112, -2113,2114,2115,2116,2117, 804,2118,2119,1230,1231, 805,1456, 405,1136,2120,2121, -2122,2123,2124, 720, 701,1297, 992,1457, 927,1004,2125,2126,2127,2128,2129,2130, - 22, 417,2131, 303,2132, 385,2133, 971, 520, 513,2134,1174, 73,1096, 231, 274, - 962,1458, 673,2135,1459,2136, 152,1137,2137,2138,2139,2140,1005,1138,1460,1139, -2141,2142,2143,2144, 11, 374, 844,2145, 154,1232, 46,1461,2146, 838, 830, 721, -1233, 106,2147, 90, 428, 462, 578, 566,1175, 352,2148,2149, 538,1234, 124,1298, -2150,1462, 761, 565,2151, 686,2152, 649,2153, 72, 173,2154, 460, 415,2155,1463, -2156,1235, 305,2157,2158,2159,2160,2161,2162, 579,2163,2164,2165,2166,2167, 747, -2168,2169,2170,2171,1464, 669,2172,2173,2174,2175,2176,1465,2177, 23, 530, 285, -2178, 335, 729,2179, 397,2180,2181,2182,1030,2183,2184, 698,2185,2186, 325,2187, -2188, 369,2189, 799,1097,1015, 348,2190,1069, 680,2191, 851,1466,2192,2193, 10, -2194, 613, 424,2195, 979, 108, 449, 589, 27, 172, 81,1031, 80, 774, 281, 350, -1032, 525, 301, 582,1176,2196, 674,1045,2197,2198,1467, 730, 762,2199,2200,2201, -2202,1468,2203, 993,2204,2205, 266,1070, 963,1140,2206,2207,2208, 664,1098, 972, -2209,2210,2211,1177,1469,1470, 871,2212,2213,2214,2215,2216,1471,2217,2218,2219, -2220,2221,2222,2223,2224,2225,2226,2227,1472,1236,2228,2229,2230,2231,2232,2233, -2234,2235,1299,2236,2237, 200,2238, 477, 373,2239,2240, 731, 825, 777,2241,2242, -2243, 521, 486, 548,2244,2245,2246,1473,1300, 53, 549, 137, 875, 76, 158,2247, -1301,1474, 469, 396,1016, 278, 712,2248, 321, 442, 503, 767, 744, 941,1237,1178, -1475,2249, 82, 178,1141,1179, 973,2250,1302,2251, 297,2252,2253, 570,2254,2255, -2256, 18, 450, 206,2257, 290, 292,1142,2258, 511, 162, 99, 346, 164, 735,2259, -1476,1477, 4, 554, 343, 798,1099,2260,1100,2261, 43, 171,1303, 139, 215,2262, -2263, 717, 775,2264,1033, 322, 216,2265, 831,2266, 149,2267,1304,2268,2269, 702, -1238, 135, 845, 347, 309,2270, 484,2271, 878, 655, 238,1006,1478,2272, 67,2273, - 295,2274,2275, 461,2276, 478, 942, 412,2277,1034,2278,2279,2280, 265,2281, 541, -2282,2283,2284,2285,2286, 70, 852,1071,2287,2288,2289,2290, 21, 56, 509, 117, - 432,2291,2292, 331, 980, 552,1101, 148, 284, 105, 393,1180,1239, 755,2293, 187, -2294,1046,1479,2295, 340,2296, 63,1047, 230,2297,2298,1305, 763,1306, 101, 800, - 808, 494,2299,2300,2301, 903,2302, 37,1072, 14, 5,2303, 79, 675,2304, 312, -2305,2306,2307,2308,2309,1480, 6,1307,2310,2311,2312, 1, 470, 35, 24, 229, -2313, 695, 210, 86, 778, 15, 784, 592, 779, 32, 77, 855, 964,2314, 259,2315, - 501, 380,2316,2317, 83, 981, 153, 689,1308,1481,1482,1483,2318,2319, 716,1484, -2320,2321,2322,2323,2324,2325,1485,2326,2327, 128, 57, 68, 261,1048, 211, 170, -1240, 31,2328, 51, 435, 742,2329,2330,2331, 635,2332, 264, 456,2333,2334,2335, - 425,2336,1486, 143, 507, 263, 943,2337, 363, 920,1487, 256,1488,1102, 243, 601, -1489,2338,2339,2340,2341,2342,2343,2344, 861,2345,2346,2347,2348,2349,2350, 395, -2351,1490,1491, 62, 535, 166, 225,2352,2353, 668, 419,1241, 138, 604, 928,2354, -1181,2355,1492,1493,2356,2357,2358,1143,2359, 696,2360, 387, 307,1309, 682, 476, -2361,2362, 332, 12, 222, 156,2363, 232,2364, 641, 276, 656, 517,1494,1495,1035, - 416, 736,1496,2365,1017, 586,2366,2367,2368,1497,2369, 242,2370,2371,2372,1498, -2373, 965, 713,2374,2375,2376,2377, 740, 982,1499, 944,1500,1007,2378,2379,1310, -1501,2380,2381,2382, 785, 329,2383,2384,1502,2385,2386,2387, 932,2388,1503,2389, -2390,2391,2392,1242,2393,2394,2395,2396,2397, 994, 950,2398,2399,2400,2401,1504, -1311,2402,2403,2404,2405,1049, 749,2406,2407, 853, 718,1144,1312,2408,1182,1505, -2409,2410, 255, 516, 479, 564, 550, 214,1506,1507,1313, 413, 239, 444, 339,1145, -1036,1508,1509,1314,1037,1510,1315,2411,1511,2412,2413,2414, 176, 703, 497, 624, - 593, 921, 302,2415, 341, 165,1103,1512,2416,1513,2417,2418,2419, 376,2420, 700, -2421,2422,2423, 258, 768,1316,2424,1183,2425, 995, 608,2426,2427,2428,2429, 221, -2430,2431,2432,2433,2434,2435,2436,2437, 195, 323, 726, 188, 897, 983,1317, 377, - 644,1050, 879,2438, 452,2439,2440,2441,2442,2443,2444, 914,2445,2446,2447,2448, - 915, 489,2449,1514,1184,2450,2451, 515, 64, 427, 495,2452, 583,2453, 483, 485, -1038, 562, 213,1515, 748, 666,2454,2455,2456,2457, 334,2458, 780, 996,1008, 705, -1243,2459,2460,2461,2462,2463, 114,2464, 493,1146, 366, 163,1516, 961,1104,2465, - 291,2466,1318,1105,2467,1517, 365,2468, 355, 951,1244,2469,1319,2470, 631,2471, -2472, 218,1320, 364, 320, 756,1518,1519,1321,1520,1322,2473,2474,2475,2476, 997, -2477,2478,2479,2480, 665,1185,2481, 916,1521,2482,2483,2484, 584, 684,2485,2486, - 797,2487,1051,1186,2488,2489,2490,1522,2491,2492, 370,2493,1039,1187, 65,2494, - 434, 205, 463,1188,2495, 125, 812, 391, 402, 826, 699, 286, 398, 155, 781, 771, - 585,2496, 590, 505,1073,2497, 599, 244, 219, 917,1018, 952, 646,1523,2498,1323, -2499,2500, 49, 984, 354, 741,2501, 625,2502,1324,2503,1019, 190, 357, 757, 491, - 95, 782, 868,2504,2505,2506,2507,2508,2509, 134,1524,1074, 422,1525, 898,2510, - 161,2511,2512,2513,2514, 769,2515,1526,2516,2517, 411,1325,2518, 472,1527,2519, -2520,2521,2522,2523,2524, 985,2525,2526,2527,2528,2529,2530, 764,2531,1245,2532, -2533, 25, 204, 311,2534, 496,2535,1052,2536,2537,2538,2539,2540,2541,2542, 199, - 704, 504, 468, 758, 657,1528, 196, 44, 839,1246, 272, 750,2543, 765, 862,2544, -2545,1326,2546, 132, 615, 933,2547, 732,2548,2549,2550,1189,1529,2551, 283,1247, -1053, 607, 929,2552,2553,2554, 930, 183, 872, 616,1040,1147,2555,1148,1020, 441, - 249,1075,2556,2557,2558, 466, 743,2559,2560,2561, 92, 514, 426, 420, 526,2562, -2563,2564,2565,2566,2567,2568, 185,2569,2570,2571,2572, 776,1530, 658,2573, 362, -2574, 361, 922,1076, 793,2575,2576,2577,2578,2579,2580,1531, 251,2581,2582,2583, -2584,1532, 54, 612, 237,1327,2585,2586, 275, 408, 647, 111,2587,1533,1106, 465, - 3, 458, 9, 38,2588, 107, 110, 890, 209, 26, 737, 498,2589,1534,2590, 431, - 202, 88,1535, 356, 287,1107, 660,1149,2591, 381,1536, 986,1150, 445,1248,1151, - 974,2592,2593, 846,2594, 446, 953, 184,1249,1250, 727,2595, 923, 193, 883,2596, -2597,2598, 102, 324, 539, 817,2599, 421,1041,2600, 832,2601, 94, 175, 197, 406, -2602, 459,2603,2604,2605,2606,2607, 330, 555,2608,2609,2610, 706,1108, 389,2611, -2612,2613,2614, 233,2615, 833, 558, 931, 954,1251,2616,2617,1537, 546,2618,2619, -1009,2620,2621,2622,1538, 690,1328,2623, 955,2624,1539,2625,2626, 772,2627,2628, -2629,2630,2631, 924, 648, 863, 603,2632,2633, 934,1540, 864, 865,2634, 642,1042, - 670,1190,2635,2636,2637,2638, 168,2639, 652, 873, 542,1054,1541,2640,2641,2642, //512, 256 - -/*************************************************************************************** - *Everything below is of no interest for detection purpose * - *************************************************************************************** - -2643,2644,2645,2646,2647,2648,2649,2650,2651,2652,2653,2654,2655,2656,2657,2658, -2659,2660,2661,2662,2663,2664,2665,2666,2667,2668,2669,2670,2671,2672,2673,2674, -2675,2676,2677,2678,2679,2680,2681,2682,2683,2684,2685,2686,2687,2688,2689,2690, -2691,2692,2693,2694,2695,2696,2697,2698,2699,1542, 880,2700,2701,2702,2703,2704, -2705,2706,2707,2708,2709,2710,2711,2712,2713,2714,2715,2716,2717,2718,2719,2720, -2721,2722,2723,2724,2725,1543,2726,2727,2728,2729,2730,2731,2732,1544,2733,2734, -2735,2736,2737,2738,2739,2740,2741,2742,2743,2744,2745,2746,2747,2748,2749,2750, -2751,2752,2753,2754,1545,2755,2756,2757,2758,2759,2760,2761,2762,2763,2764,2765, -2766,1546,2767,1547,2768,2769,2770,2771,2772,2773,2774,2775,2776,2777,2778,2779, -2780,2781,2782,2783,2784,2785,2786,1548,2787,2788,2789,1109,2790,2791,2792,2793, -2794,2795,2796,2797,2798,2799,2800,2801,2802,2803,2804,2805,2806,2807,2808,2809, -2810,2811,2812,1329,2813,2814,2815,2816,2817,2818,2819,2820,2821,2822,2823,2824, -2825,2826,2827,2828,2829,2830,2831,2832,2833,2834,2835,2836,2837,2838,2839,2840, -2841,2842,2843,2844,2845,2846,2847,2848,2849,2850,2851,2852,2853,2854,2855,2856, -1549,2857,2858,2859,2860,1550,2861,2862,1551,2863,2864,2865,2866,2867,2868,2869, -2870,2871,2872,2873,2874,1110,1330,2875,2876,2877,2878,2879,2880,2881,2882,2883, -2884,2885,2886,2887,2888,2889,2890,2891,2892,2893,2894,2895,2896,2897,2898,2899, -2900,2901,2902,2903,2904,2905,2906,2907,2908,2909,2910,2911,2912,2913,2914,2915, -2916,2917,2918,2919,2920,2921,2922,2923,2924,2925,2926,2927,2928,2929,2930,1331, -2931,2932,2933,2934,2935,2936,2937,2938,2939,2940,2941,2942,2943,1552,2944,2945, -2946,2947,2948,2949,2950,2951,2952,2953,2954,2955,2956,2957,2958,2959,2960,2961, -2962,2963,2964,1252,2965,2966,2967,2968,2969,2970,2971,2972,2973,2974,2975,2976, -2977,2978,2979,2980,2981,2982,2983,2984,2985,2986,2987,2988,2989,2990,2991,2992, -2993,2994,2995,2996,2997,2998,2999,3000,3001,3002,3003,3004,3005,3006,3007,3008, -3009,3010,3011,3012,1553,3013,3014,3015,3016,3017,1554,3018,1332,3019,3020,3021, -3022,3023,3024,3025,3026,3027,3028,3029,3030,3031,3032,3033,3034,3035,3036,3037, -3038,3039,3040,3041,3042,3043,3044,3045,3046,3047,3048,3049,3050,1555,3051,3052, -3053,1556,1557,3054,3055,3056,3057,3058,3059,3060,3061,3062,3063,3064,3065,3066, -3067,1558,3068,3069,3070,3071,3072,3073,3074,3075,3076,1559,3077,3078,3079,3080, -3081,3082,3083,1253,3084,3085,3086,3087,3088,3089,3090,3091,3092,3093,3094,3095, -3096,3097,3098,3099,3100,3101,3102,3103,3104,3105,3106,3107,3108,1152,3109,3110, -3111,3112,3113,1560,3114,3115,3116,3117,1111,3118,3119,3120,3121,3122,3123,3124, -3125,3126,3127,3128,3129,3130,3131,3132,3133,3134,3135,3136,3137,3138,3139,3140, -3141,3142,3143,3144,3145,3146,3147,3148,3149,3150,3151,3152,3153,3154,3155,3156, -3157,3158,3159,3160,3161,3162,3163,3164,3165,3166,3167,3168,3169,3170,3171,3172, -3173,3174,3175,3176,1333,3177,3178,3179,3180,3181,3182,3183,3184,3185,3186,3187, -3188,3189,1561,3190,3191,1334,3192,3193,3194,3195,3196,3197,3198,3199,3200,3201, -3202,3203,3204,3205,3206,3207,3208,3209,3210,3211,3212,3213,3214,3215,3216,3217, -3218,3219,3220,3221,3222,3223,3224,3225,3226,3227,3228,3229,3230,3231,3232,3233, -3234,1562,3235,3236,3237,3238,3239,3240,3241,3242,3243,3244,3245,3246,3247,3248, -3249,3250,3251,3252,3253,3254,3255,3256,3257,3258,3259,3260,3261,3262,3263,3264, -3265,3266,3267,3268,3269,3270,3271,3272,3273,3274,3275,3276,3277,1563,3278,3279, -3280,3281,3282,3283,3284,3285,3286,3287,3288,3289,3290,3291,3292,3293,3294,3295, -3296,3297,3298,3299,3300,3301,3302,3303,3304,3305,3306,3307,3308,3309,3310,3311, -3312,3313,3314,3315,3316,3317,3318,3319,3320,3321,3322,3323,3324,3325,3326,3327, -3328,3329,3330,3331,3332,3333,3334,3335,3336,3337,3338,3339,3340,3341,3342,3343, -3344,3345,3346,3347,3348,3349,3350,3351,3352,3353,3354,3355,3356,3357,3358,3359, -3360,3361,3362,3363,3364,1335,3365,3366,3367,3368,3369,3370,3371,3372,3373,3374, -3375,3376,3377,3378,3379,3380,3381,3382,3383,3384,3385,3386,3387,1336,3388,3389, -3390,3391,3392,3393,3394,3395,3396,3397,3398,3399,3400,3401,3402,3403,3404,3405, -3406,3407,3408,3409,3410,3411,3412,3413,3414,1337,3415,3416,3417,3418,3419,1338, -3420,3421,3422,1564,1565,3423,3424,3425,3426,3427,3428,3429,3430,3431,1254,3432, -3433,3434,1339,3435,3436,3437,3438,3439,1566,3440,3441,3442,3443,3444,3445,3446, -3447,3448,3449,3450,3451,3452,3453,3454,1255,3455,3456,3457,3458,3459,1567,1191, -3460,1568,1569,3461,3462,3463,1570,3464,3465,3466,3467,3468,1571,3469,3470,3471, -3472,3473,1572,3474,3475,3476,3477,3478,3479,3480,3481,3482,3483,3484,3485,3486, -1340,3487,3488,3489,3490,3491,3492,1021,3493,3494,3495,3496,3497,3498,1573,3499, -1341,3500,3501,3502,3503,3504,3505,3506,3507,3508,3509,3510,3511,1342,3512,3513, -3514,3515,3516,1574,1343,3517,3518,3519,1575,3520,1576,3521,3522,3523,3524,3525, -3526,3527,3528,3529,3530,3531,3532,3533,3534,3535,3536,3537,3538,3539,3540,3541, -3542,3543,3544,3545,3546,3547,3548,3549,3550,3551,3552,3553,3554,3555,3556,3557, -3558,3559,3560,3561,3562,3563,3564,3565,3566,3567,3568,3569,3570,3571,3572,3573, -3574,3575,3576,3577,3578,3579,3580,1577,3581,3582,1578,3583,3584,3585,3586,3587, -3588,3589,3590,3591,3592,3593,3594,3595,3596,3597,3598,3599,3600,3601,3602,3603, -3604,1579,3605,3606,3607,3608,3609,3610,3611,3612,3613,3614,3615,3616,3617,3618, -3619,3620,3621,3622,3623,3624,3625,3626,3627,3628,3629,1580,3630,3631,1581,3632, -3633,3634,3635,3636,3637,3638,3639,3640,3641,3642,3643,3644,3645,3646,3647,3648, -3649,3650,3651,3652,3653,3654,3655,3656,1582,3657,3658,3659,3660,3661,3662,3663, -3664,3665,3666,3667,3668,3669,3670,3671,3672,3673,3674,3675,3676,3677,3678,3679, -3680,3681,3682,3683,3684,3685,3686,3687,3688,3689,3690,3691,3692,3693,3694,3695, -3696,3697,3698,3699,3700,1192,3701,3702,3703,3704,1256,3705,3706,3707,3708,1583, -1257,3709,3710,3711,3712,3713,3714,3715,3716,1584,3717,3718,3719,3720,3721,3722, -3723,3724,3725,3726,3727,3728,3729,3730,3731,3732,3733,3734,3735,3736,3737,3738, -3739,3740,3741,3742,3743,3744,3745,1344,3746,3747,3748,3749,3750,3751,3752,3753, -3754,3755,3756,1585,3757,3758,3759,3760,3761,3762,3763,3764,3765,3766,1586,3767, -3768,3769,3770,3771,3772,3773,3774,3775,3776,3777,3778,1345,3779,3780,3781,3782, -3783,3784,3785,3786,3787,3788,3789,3790,3791,3792,3793,3794,3795,1346,1587,3796, -3797,1588,3798,3799,3800,3801,3802,3803,3804,3805,3806,1347,3807,3808,3809,3810, -3811,1589,3812,3813,3814,3815,3816,3817,3818,3819,3820,3821,1590,3822,3823,1591, -1348,3824,3825,3826,3827,3828,3829,3830,1592,3831,3832,1593,3833,3834,3835,3836, -3837,3838,3839,3840,3841,3842,3843,3844,1349,3845,3846,3847,3848,3849,3850,3851, -3852,3853,3854,3855,3856,3857,3858,1594,3859,3860,3861,3862,3863,3864,3865,3866, -3867,3868,3869,1595,3870,3871,3872,3873,1596,3874,3875,3876,3877,3878,3879,3880, -3881,3882,3883,3884,3885,3886,1597,3887,3888,3889,3890,3891,3892,3893,3894,3895, -1598,3896,3897,3898,1599,1600,3899,1350,3900,1351,3901,3902,1352,3903,3904,3905, -3906,3907,3908,3909,3910,3911,3912,3913,3914,3915,3916,3917,3918,3919,3920,3921, -3922,3923,3924,1258,3925,3926,3927,3928,3929,3930,3931,1193,3932,1601,3933,3934, -3935,3936,3937,3938,3939,3940,3941,3942,3943,1602,3944,3945,3946,3947,3948,1603, -3949,3950,3951,3952,3953,3954,3955,3956,3957,3958,3959,3960,3961,3962,3963,3964, -3965,1604,3966,3967,3968,3969,3970,3971,3972,3973,3974,3975,3976,3977,1353,3978, -3979,3980,3981,3982,3983,3984,3985,3986,3987,3988,3989,3990,3991,1354,3992,3993, -3994,3995,3996,3997,3998,3999,4000,4001,4002,4003,4004,4005,4006,4007,4008,4009, -4010,4011,4012,4013,4014,4015,4016,4017,4018,4019,4020,4021,4022,4023,1355,4024, -4025,4026,4027,4028,4029,4030,4031,4032,4033,4034,4035,4036,4037,4038,4039,4040, -1605,4041,4042,4043,4044,4045,4046,4047,4048,4049,4050,4051,4052,4053,4054,4055, -4056,4057,4058,4059,4060,1606,4061,4062,4063,4064,1607,4065,4066,4067,4068,4069, -4070,4071,4072,4073,4074,4075,4076,1194,4077,4078,1608,4079,4080,4081,4082,4083, -4084,4085,4086,4087,1609,4088,4089,4090,4091,4092,4093,4094,4095,4096,4097,4098, -4099,4100,4101,4102,4103,4104,4105,4106,4107,4108,1259,4109,4110,4111,4112,4113, -4114,4115,4116,4117,4118,4119,4120,4121,4122,4123,4124,1195,4125,4126,4127,1610, -4128,4129,4130,4131,4132,4133,4134,4135,4136,4137,1356,4138,4139,4140,4141,4142, -4143,4144,1611,4145,4146,4147,4148,4149,4150,4151,4152,4153,4154,4155,4156,4157, -4158,4159,4160,4161,4162,4163,4164,4165,4166,4167,4168,4169,4170,4171,4172,4173, -4174,4175,4176,4177,4178,4179,4180,4181,4182,4183,4184,4185,4186,4187,4188,4189, -4190,4191,4192,4193,4194,4195,4196,4197,4198,4199,4200,4201,4202,4203,4204,4205, -4206,4207,4208,4209,4210,4211,4212,4213,4214,4215,4216,4217,4218,4219,1612,4220, -4221,4222,4223,4224,4225,4226,4227,1357,4228,1613,4229,4230,4231,4232,4233,4234, -4235,4236,4237,4238,4239,4240,4241,4242,4243,1614,4244,4245,4246,4247,4248,4249, -4250,4251,4252,4253,4254,4255,4256,4257,4258,4259,4260,4261,4262,4263,4264,4265, -4266,4267,4268,4269,4270,1196,1358,4271,4272,4273,4274,4275,4276,4277,4278,4279, -4280,4281,4282,4283,4284,4285,4286,4287,1615,4288,4289,4290,4291,4292,4293,4294, -4295,4296,4297,4298,4299,4300,4301,4302,4303,4304,4305,4306,4307,4308,4309,4310, -4311,4312,4313,4314,4315,4316,4317,4318,4319,4320,4321,4322,4323,4324,4325,4326, -4327,4328,4329,4330,4331,4332,4333,4334,1616,4335,4336,4337,4338,4339,4340,4341, -4342,4343,4344,4345,4346,4347,4348,4349,4350,4351,4352,4353,4354,4355,4356,4357, -4358,4359,4360,1617,4361,4362,4363,4364,4365,1618,4366,4367,4368,4369,4370,4371, -4372,4373,4374,4375,4376,4377,4378,4379,4380,4381,4382,4383,4384,4385,4386,4387, -4388,4389,4390,4391,4392,4393,4394,4395,4396,4397,4398,4399,4400,4401,4402,4403, -4404,4405,4406,4407,4408,4409,4410,4411,4412,4413,4414,4415,4416,1619,4417,4418, -4419,4420,4421,4422,4423,4424,4425,1112,4426,4427,4428,4429,4430,1620,4431,4432, -4433,4434,4435,4436,4437,4438,4439,4440,4441,4442,1260,1261,4443,4444,4445,4446, -4447,4448,4449,4450,4451,4452,4453,4454,4455,1359,4456,4457,4458,4459,4460,4461, -4462,4463,4464,4465,1621,4466,4467,4468,4469,4470,4471,4472,4473,4474,4475,4476, -4477,4478,4479,4480,4481,4482,4483,4484,4485,4486,4487,4488,4489,1055,4490,4491, -4492,4493,4494,4495,4496,4497,4498,4499,4500,4501,4502,4503,4504,4505,4506,4507, -4508,4509,4510,4511,4512,4513,4514,4515,4516,4517,4518,1622,4519,4520,4521,1623, -4522,4523,4524,4525,4526,4527,4528,4529,4530,4531,4532,4533,4534,4535,1360,4536, -4537,4538,4539,4540,4541,4542,4543, 975,4544,4545,4546,4547,4548,4549,4550,4551, -4552,4553,4554,4555,4556,4557,4558,4559,4560,4561,4562,4563,4564,4565,4566,4567, -4568,4569,4570,4571,1624,4572,4573,4574,4575,4576,1625,4577,4578,4579,4580,4581, -4582,4583,4584,1626,4585,4586,4587,4588,4589,4590,4591,4592,4593,4594,4595,1627, -4596,4597,4598,4599,4600,4601,4602,4603,4604,4605,4606,4607,4608,4609,4610,4611, -4612,4613,4614,4615,1628,4616,4617,4618,4619,4620,4621,4622,4623,4624,4625,4626, -4627,4628,4629,4630,4631,4632,4633,4634,4635,4636,4637,4638,4639,4640,4641,4642, -4643,4644,4645,4646,4647,4648,4649,1361,4650,4651,4652,4653,4654,4655,4656,4657, -4658,4659,4660,4661,1362,4662,4663,4664,4665,4666,4667,4668,4669,4670,4671,4672, -4673,4674,4675,4676,4677,4678,4679,4680,4681,4682,1629,4683,4684,4685,4686,4687, -1630,4688,4689,4690,4691,1153,4692,4693,4694,1113,4695,4696,4697,4698,4699,4700, -4701,4702,4703,4704,4705,4706,4707,4708,4709,4710,4711,1197,4712,4713,4714,4715, -4716,4717,4718,4719,4720,4721,4722,4723,4724,4725,4726,4727,4728,4729,4730,4731, -4732,4733,4734,4735,1631,4736,1632,4737,4738,4739,4740,4741,4742,4743,4744,1633, -4745,4746,4747,4748,4749,1262,4750,4751,4752,4753,4754,1363,4755,4756,4757,4758, -4759,4760,4761,4762,4763,4764,4765,4766,4767,4768,1634,4769,4770,4771,4772,4773, -4774,4775,4776,4777,4778,1635,4779,4780,4781,4782,4783,4784,4785,4786,4787,4788, -4789,1636,4790,4791,4792,4793,4794,4795,4796,4797,4798,4799,4800,4801,4802,4803, -4804,4805,4806,1637,4807,4808,4809,1638,4810,4811,4812,4813,4814,4815,4816,4817, -4818,1639,4819,4820,4821,4822,4823,4824,4825,4826,4827,4828,4829,4830,4831,4832, -4833,1077,4834,4835,4836,4837,4838,4839,4840,4841,4842,4843,4844,4845,4846,4847, -4848,4849,4850,4851,4852,4853,4854,4855,4856,4857,4858,4859,4860,4861,4862,4863, -4864,4865,4866,4867,4868,4869,4870,4871,4872,4873,4874,4875,4876,4877,4878,4879, -4880,4881,4882,4883,1640,4884,4885,1641,4886,4887,4888,4889,4890,4891,4892,4893, -4894,4895,4896,4897,4898,4899,4900,4901,4902,4903,4904,4905,4906,4907,4908,4909, -4910,4911,1642,4912,4913,4914,1364,4915,4916,4917,4918,4919,4920,4921,4922,4923, -4924,4925,4926,4927,4928,4929,4930,4931,1643,4932,4933,4934,4935,4936,4937,4938, -4939,4940,4941,4942,4943,4944,4945,4946,4947,4948,4949,4950,4951,4952,4953,4954, -4955,4956,4957,4958,4959,4960,4961,4962,4963,4964,4965,4966,4967,4968,4969,4970, -4971,4972,4973,4974,4975,4976,4977,4978,4979,4980,1644,4981,4982,4983,4984,1645, -4985,4986,1646,4987,4988,4989,4990,4991,4992,4993,4994,4995,4996,4997,4998,4999, -5000,5001,5002,5003,5004,5005,1647,5006,1648,5007,5008,5009,5010,5011,5012,1078, -5013,5014,5015,5016,5017,5018,5019,5020,5021,5022,5023,5024,5025,5026,5027,5028, -1365,5029,5030,5031,5032,5033,5034,5035,5036,5037,5038,5039,1649,5040,5041,5042, -5043,5044,5045,1366,5046,5047,5048,5049,5050,5051,5052,5053,5054,5055,1650,5056, -5057,5058,5059,5060,5061,5062,5063,5064,5065,5066,5067,5068,5069,5070,5071,5072, -5073,5074,5075,5076,5077,1651,5078,5079,5080,5081,5082,5083,5084,5085,5086,5087, -5088,5089,5090,5091,5092,5093,5094,5095,5096,5097,5098,5099,5100,5101,5102,5103, -5104,5105,5106,5107,5108,5109,5110,1652,5111,5112,5113,5114,5115,5116,5117,5118, -1367,5119,5120,5121,5122,5123,5124,5125,5126,5127,5128,5129,1653,5130,5131,5132, -5133,5134,5135,5136,5137,5138,5139,5140,5141,5142,5143,5144,5145,5146,5147,5148, -5149,1368,5150,1654,5151,1369,5152,5153,5154,5155,5156,5157,5158,5159,5160,5161, -5162,5163,5164,5165,5166,5167,5168,5169,5170,5171,5172,5173,5174,5175,5176,5177, -5178,1370,5179,5180,5181,5182,5183,5184,5185,5186,5187,5188,5189,5190,5191,5192, -5193,5194,5195,5196,5197,5198,1655,5199,5200,5201,5202,1656,5203,5204,5205,5206, -1371,5207,1372,5208,5209,5210,5211,1373,5212,5213,1374,5214,5215,5216,5217,5218, -5219,5220,5221,5222,5223,5224,5225,5226,5227,5228,5229,5230,5231,5232,5233,5234, -5235,5236,5237,5238,5239,5240,5241,5242,5243,5244,5245,5246,5247,1657,5248,5249, -5250,5251,1658,1263,5252,5253,5254,5255,5256,1375,5257,5258,5259,5260,5261,5262, -5263,5264,5265,5266,5267,5268,5269,5270,5271,5272,5273,5274,5275,5276,5277,5278, -5279,5280,5281,5282,5283,1659,5284,5285,5286,5287,5288,5289,5290,5291,5292,5293, -5294,5295,5296,5297,5298,5299,5300,1660,5301,5302,5303,5304,5305,5306,5307,5308, -5309,5310,5311,5312,5313,5314,5315,5316,5317,5318,5319,5320,5321,1376,5322,5323, -5324,5325,5326,5327,5328,5329,5330,5331,5332,5333,1198,5334,5335,5336,5337,5338, -5339,5340,5341,5342,5343,1661,5344,5345,5346,5347,5348,5349,5350,5351,5352,5353, -5354,5355,5356,5357,5358,5359,5360,5361,5362,5363,5364,5365,5366,5367,5368,5369, -5370,5371,5372,5373,5374,5375,5376,5377,5378,5379,5380,5381,5382,5383,5384,5385, -5386,5387,5388,5389,5390,5391,5392,5393,5394,5395,5396,5397,5398,1264,5399,5400, -5401,5402,5403,5404,5405,5406,5407,5408,5409,5410,5411,5412,1662,5413,5414,5415, -5416,1663,5417,5418,5419,5420,5421,5422,5423,5424,5425,5426,5427,5428,5429,5430, -5431,5432,5433,5434,5435,5436,5437,5438,1664,5439,5440,5441,5442,5443,5444,5445, -5446,5447,5448,5449,5450,5451,5452,5453,5454,5455,5456,5457,5458,5459,5460,5461, -5462,5463,5464,5465,5466,5467,5468,5469,5470,5471,5472,5473,5474,5475,5476,5477, -5478,1154,5479,5480,5481,5482,5483,5484,5485,1665,5486,5487,5488,5489,5490,5491, -5492,5493,5494,5495,5496,5497,5498,5499,5500,5501,5502,5503,5504,5505,5506,5507, -5508,5509,5510,5511,5512,5513,5514,5515,5516,5517,5518,5519,5520,5521,5522,5523, -5524,5525,5526,5527,5528,5529,5530,5531,5532,5533,5534,5535,5536,5537,5538,5539, -5540,5541,5542,5543,5544,5545,5546,5547,5548,1377,5549,5550,5551,5552,5553,5554, -5555,5556,5557,5558,5559,5560,5561,5562,5563,5564,5565,5566,5567,5568,5569,5570, -1114,5571,5572,5573,5574,5575,5576,5577,5578,5579,5580,5581,5582,5583,5584,5585, -5586,5587,5588,5589,5590,5591,5592,1378,5593,5594,5595,5596,5597,5598,5599,5600, -5601,5602,5603,5604,5605,5606,5607,5608,5609,5610,5611,5612,5613,5614,1379,5615, -5616,5617,5618,5619,5620,5621,5622,5623,5624,5625,5626,5627,5628,5629,5630,5631, -5632,5633,5634,1380,5635,5636,5637,5638,5639,5640,5641,5642,5643,5644,5645,5646, -5647,5648,5649,1381,1056,5650,5651,5652,5653,5654,5655,5656,5657,5658,5659,5660, -1666,5661,5662,5663,5664,5665,5666,5667,5668,1667,5669,1668,5670,5671,5672,5673, -5674,5675,5676,5677,5678,1155,5679,5680,5681,5682,5683,5684,5685,5686,5687,5688, -5689,5690,5691,5692,5693,5694,5695,5696,5697,5698,1669,5699,5700,5701,5702,5703, -5704,5705,1670,5706,5707,5708,5709,5710,1671,5711,5712,5713,5714,1382,5715,5716, -5717,5718,5719,5720,5721,5722,5723,5724,5725,1672,5726,5727,1673,1674,5728,5729, -5730,5731,5732,5733,5734,5735,5736,1675,5737,5738,5739,5740,5741,5742,5743,5744, -1676,5745,5746,5747,5748,5749,5750,5751,1383,5752,5753,5754,5755,5756,5757,5758, -5759,5760,5761,5762,5763,5764,5765,5766,5767,5768,1677,5769,5770,5771,5772,5773, -1678,5774,5775,5776, 998,5777,5778,5779,5780,5781,5782,5783,5784,5785,1384,5786, -5787,5788,5789,5790,5791,5792,5793,5794,5795,5796,5797,5798,5799,5800,1679,5801, -5802,5803,1115,1116,5804,5805,5806,5807,5808,5809,5810,5811,5812,5813,5814,5815, -5816,5817,5818,5819,5820,5821,5822,5823,5824,5825,5826,5827,5828,5829,5830,5831, -5832,5833,5834,5835,5836,5837,5838,5839,5840,5841,5842,5843,5844,5845,5846,5847, -5848,5849,5850,5851,5852,5853,5854,5855,1680,5856,5857,5858,5859,5860,5861,5862, -5863,5864,1681,5865,5866,5867,1682,5868,5869,5870,5871,5872,5873,5874,5875,5876, -5877,5878,5879,1683,5880,1684,5881,5882,5883,5884,1685,5885,5886,5887,5888,5889, -5890,5891,5892,5893,5894,5895,5896,5897,5898,5899,5900,5901,5902,5903,5904,5905, -5906,5907,1686,5908,5909,5910,5911,5912,5913,5914,5915,5916,5917,5918,5919,5920, -5921,5922,5923,5924,5925,5926,5927,5928,5929,5930,5931,5932,5933,5934,5935,1687, -5936,5937,5938,5939,5940,5941,5942,5943,5944,5945,5946,5947,5948,5949,5950,5951, -5952,1688,1689,5953,1199,5954,5955,5956,5957,5958,5959,5960,5961,1690,5962,5963, -5964,5965,5966,5967,5968,5969,5970,5971,5972,5973,5974,5975,5976,5977,5978,5979, -5980,5981,1385,5982,1386,5983,5984,5985,5986,5987,5988,5989,5990,5991,5992,5993, -5994,5995,5996,5997,5998,5999,6000,6001,6002,6003,6004,6005,6006,6007,6008,6009, -6010,6011,6012,6013,6014,6015,6016,6017,6018,6019,6020,6021,6022,6023,6024,6025, -6026,6027,1265,6028,6029,1691,6030,6031,6032,6033,6034,6035,6036,6037,6038,6039, -6040,6041,6042,6043,6044,6045,6046,6047,6048,6049,6050,6051,6052,6053,6054,6055, -6056,6057,6058,6059,6060,6061,6062,6063,6064,6065,6066,6067,6068,6069,6070,6071, -6072,6073,6074,6075,6076,6077,6078,6079,6080,6081,6082,6083,6084,1692,6085,6086, -6087,6088,6089,6090,6091,6092,6093,6094,6095,6096,6097,6098,6099,6100,6101,6102, -6103,6104,6105,6106,6107,6108,6109,6110,6111,6112,6113,6114,6115,6116,6117,6118, -6119,6120,6121,6122,6123,6124,6125,6126,6127,6128,6129,6130,6131,1693,6132,6133, -6134,6135,6136,1694,6137,6138,6139,6140,6141,1695,6142,6143,6144,6145,6146,6147, -6148,6149,6150,6151,6152,6153,6154,6155,6156,6157,6158,6159,6160,6161,6162,6163, -6164,6165,6166,6167,6168,6169,6170,6171,6172,6173,6174,6175,6176,6177,6178,6179, -6180,6181,6182,6183,6184,6185,1696,6186,6187,6188,6189,6190,6191,6192,6193,6194, -6195,6196,6197,6198,6199,6200,6201,6202,6203,6204,6205,6206,6207,6208,6209,6210, -6211,6212,6213,6214,6215,6216,6217,6218,6219,1697,6220,6221,6222,6223,6224,6225, -6226,6227,6228,6229,6230,6231,6232,6233,6234,6235,6236,6237,6238,6239,6240,6241, -6242,6243,6244,6245,6246,6247,6248,6249,6250,6251,6252,6253,1698,6254,6255,6256, -6257,6258,6259,6260,6261,6262,6263,1200,6264,6265,6266,6267,6268,6269,6270,6271, //1024 -6272,6273,6274,6275,6276,6277,6278,6279,6280,6281,6282,6283,6284,6285,6286,6287, -6288,6289,6290,6291,6292,6293,6294,6295,6296,6297,6298,6299,6300,6301,6302,1699, -6303,6304,1700,6305,6306,6307,6308,6309,6310,6311,6312,6313,6314,6315,6316,6317, -6318,6319,6320,6321,6322,6323,6324,6325,6326,6327,6328,6329,6330,6331,6332,6333, -6334,6335,6336,6337,6338,6339,1701,6340,6341,6342,6343,6344,1387,6345,6346,6347, -6348,6349,6350,6351,6352,6353,6354,6355,6356,6357,6358,6359,6360,6361,6362,6363, -6364,6365,6366,6367,6368,6369,6370,6371,6372,6373,6374,6375,6376,6377,6378,6379, -6380,6381,6382,6383,6384,6385,6386,6387,6388,6389,6390,6391,6392,6393,6394,6395, -6396,6397,6398,6399,6400,6401,6402,6403,6404,6405,6406,6407,6408,6409,6410,6411, -6412,6413,1702,6414,6415,6416,6417,6418,6419,6420,6421,6422,1703,6423,6424,6425, -6426,6427,6428,6429,6430,6431,6432,6433,6434,6435,6436,6437,6438,1704,6439,6440, -6441,6442,6443,6444,6445,6446,6447,6448,6449,6450,6451,6452,6453,6454,6455,6456, -6457,6458,6459,6460,6461,6462,6463,6464,6465,6466,6467,6468,6469,6470,6471,6472, -6473,6474,6475,6476,6477,6478,6479,6480,6481,6482,6483,6484,6485,6486,6487,6488, -6489,6490,6491,6492,6493,6494,6495,6496,6497,6498,6499,6500,6501,6502,6503,1266, -6504,6505,6506,6507,6508,6509,6510,6511,6512,6513,6514,6515,6516,6517,6518,6519, -6520,6521,6522,6523,6524,6525,6526,6527,6528,6529,6530,6531,6532,6533,6534,6535, -6536,6537,6538,6539,6540,6541,6542,6543,6544,6545,6546,6547,6548,6549,6550,6551, -1705,1706,6552,6553,6554,6555,6556,6557,6558,6559,6560,6561,6562,6563,6564,6565, -6566,6567,6568,6569,6570,6571,6572,6573,6574,6575,6576,6577,6578,6579,6580,6581, -6582,6583,6584,6585,6586,6587,6588,6589,6590,6591,6592,6593,6594,6595,6596,6597, -6598,6599,6600,6601,6602,6603,6604,6605,6606,6607,6608,6609,6610,6611,6612,6613, -6614,6615,6616,6617,6618,6619,6620,6621,6622,6623,6624,6625,6626,6627,6628,6629, -6630,6631,6632,6633,6634,6635,6636,6637,1388,6638,6639,6640,6641,6642,6643,6644, -1707,6645,6646,6647,6648,6649,6650,6651,6652,6653,6654,6655,6656,6657,6658,6659, -6660,6661,6662,6663,1708,6664,6665,6666,6667,6668,6669,6670,6671,6672,6673,6674, -1201,6675,6676,6677,6678,6679,6680,6681,6682,6683,6684,6685,6686,6687,6688,6689, -6690,6691,6692,6693,6694,6695,6696,6697,6698,6699,6700,6701,6702,6703,6704,6705, -6706,6707,6708,6709,6710,6711,6712,6713,6714,6715,6716,6717,6718,6719,6720,6721, -6722,6723,6724,6725,1389,6726,6727,6728,6729,6730,6731,6732,6733,6734,6735,6736, -1390,1709,6737,6738,6739,6740,6741,6742,1710,6743,6744,6745,6746,1391,6747,6748, -6749,6750,6751,6752,6753,6754,6755,6756,6757,1392,6758,6759,6760,6761,6762,6763, -6764,6765,6766,6767,6768,6769,6770,6771,6772,6773,6774,6775,6776,6777,6778,6779, -6780,1202,6781,6782,6783,6784,6785,6786,6787,6788,6789,6790,6791,6792,6793,6794, -6795,6796,6797,6798,6799,6800,6801,6802,6803,6804,6805,6806,6807,6808,6809,1711, -6810,6811,6812,6813,6814,6815,6816,6817,6818,6819,6820,6821,6822,6823,6824,6825, -6826,6827,6828,6829,6830,6831,6832,6833,6834,6835,6836,1393,6837,6838,6839,6840, -6841,6842,6843,6844,6845,6846,6847,6848,6849,6850,6851,6852,6853,6854,6855,6856, -6857,6858,6859,6860,6861,6862,6863,6864,6865,6866,6867,6868,6869,6870,6871,6872, -6873,6874,6875,6876,6877,6878,6879,6880,6881,6882,6883,6884,6885,6886,6887,6888, -6889,6890,6891,6892,6893,6894,6895,6896,6897,6898,6899,6900,6901,6902,1712,6903, -6904,6905,6906,6907,6908,6909,6910,1713,6911,6912,6913,6914,6915,6916,6917,6918, -6919,6920,6921,6922,6923,6924,6925,6926,6927,6928,6929,6930,6931,6932,6933,6934, -6935,6936,6937,6938,6939,6940,6941,6942,6943,6944,6945,6946,6947,6948,6949,6950, -6951,6952,6953,6954,6955,6956,6957,6958,6959,6960,6961,6962,6963,6964,6965,6966, -6967,6968,6969,6970,6971,6972,6973,6974,1714,6975,6976,6977,6978,6979,6980,6981, -6982,6983,6984,6985,6986,6987,6988,1394,6989,6990,6991,6992,6993,6994,6995,6996, -6997,6998,6999,7000,1715,7001,7002,7003,7004,7005,7006,7007,7008,7009,7010,7011, -7012,7013,7014,7015,7016,7017,7018,7019,7020,7021,7022,7023,7024,7025,7026,7027, -7028,1716,7029,7030,7031,7032,7033,7034,7035,7036,7037,7038,7039,7040,7041,7042, -7043,7044,7045,7046,7047,7048,7049,7050,7051,7052,7053,7054,7055,7056,7057,7058, -7059,7060,7061,7062,7063,7064,7065,7066,7067,7068,7069,7070,7071,7072,7073,7074, -7075,7076,7077,7078,7079,7080,7081,7082,7083,7084,7085,7086,7087,7088,7089,7090, -7091,7092,7093,7094,7095,7096,7097,7098,7099,7100,7101,7102,7103,7104,7105,7106, -7107,7108,7109,7110,7111,7112,7113,7114,7115,7116,7117,7118,7119,7120,7121,7122, -7123,7124,7125,7126,7127,7128,7129,7130,7131,7132,7133,7134,7135,7136,7137,7138, -7139,7140,7141,7142,7143,7144,7145,7146,7147,7148,7149,7150,7151,7152,7153,7154, -7155,7156,7157,7158,7159,7160,7161,7162,7163,7164,7165,7166,7167,7168,7169,7170, -7171,7172,7173,7174,7175,7176,7177,7178,7179,7180,7181,7182,7183,7184,7185,7186, -7187,7188,7189,7190,7191,7192,7193,7194,7195,7196,7197,7198,7199,7200,7201,7202, -7203,7204,7205,7206,7207,1395,7208,7209,7210,7211,7212,7213,1717,7214,7215,7216, -7217,7218,7219,7220,7221,7222,7223,7224,7225,7226,7227,7228,7229,7230,7231,7232, -7233,7234,7235,7236,7237,7238,7239,7240,7241,7242,7243,7244,7245,7246,7247,7248, -7249,7250,7251,7252,7253,7254,7255,7256,7257,7258,7259,7260,7261,7262,7263,7264, -7265,7266,7267,7268,7269,7270,7271,7272,7273,7274,7275,7276,7277,7278,7279,7280, -7281,7282,7283,7284,7285,7286,7287,7288,7289,7290,7291,7292,7293,7294,7295,7296, -7297,7298,7299,7300,7301,7302,7303,7304,7305,7306,7307,7308,7309,7310,7311,7312, -7313,1718,7314,7315,7316,7317,7318,7319,7320,7321,7322,7323,7324,7325,7326,7327, -7328,7329,7330,7331,7332,7333,7334,7335,7336,7337,7338,7339,7340,7341,7342,7343, -7344,7345,7346,7347,7348,7349,7350,7351,7352,7353,7354,7355,7356,7357,7358,7359, -7360,7361,7362,7363,7364,7365,7366,7367,7368,7369,7370,7371,7372,7373,7374,7375, -7376,7377,7378,7379,7380,7381,7382,7383,7384,7385,7386,7387,7388,7389,7390,7391, -7392,7393,7394,7395,7396,7397,7398,7399,7400,7401,7402,7403,7404,7405,7406,7407, -7408,7409,7410,7411,7412,7413,7414,7415,7416,7417,7418,7419,7420,7421,7422,7423, -7424,7425,7426,7427,7428,7429,7430,7431,7432,7433,7434,7435,7436,7437,7438,7439, -7440,7441,7442,7443,7444,7445,7446,7447,7448,7449,7450,7451,7452,7453,7454,7455, -7456,7457,7458,7459,7460,7461,7462,7463,7464,7465,7466,7467,7468,7469,7470,7471, -7472,7473,7474,7475,7476,7477,7478,7479,7480,7481,7482,7483,7484,7485,7486,7487, -7488,7489,7490,7491,7492,7493,7494,7495,7496,7497,7498,7499,7500,7501,7502,7503, -7504,7505,7506,7507,7508,7509,7510,7511,7512,7513,7514,7515,7516,7517,7518,7519, -7520,7521,7522,7523,7524,7525,7526,7527,7528,7529,7530,7531,7532,7533,7534,7535, -7536,7537,7538,7539,7540,7541,7542,7543,7544,7545,7546,7547,7548,7549,7550,7551, -7552,7553,7554,7555,7556,7557,7558,7559,7560,7561,7562,7563,7564,7565,7566,7567, -7568,7569,7570,7571,7572,7573,7574,7575,7576,7577,7578,7579,7580,7581,7582,7583, -7584,7585,7586,7587,7588,7589,7590,7591,7592,7593,7594,7595,7596,7597,7598,7599, -7600,7601,7602,7603,7604,7605,7606,7607,7608,7609,7610,7611,7612,7613,7614,7615, -7616,7617,7618,7619,7620,7621,7622,7623,7624,7625,7626,7627,7628,7629,7630,7631, -7632,7633,7634,7635,7636,7637,7638,7639,7640,7641,7642,7643,7644,7645,7646,7647, -7648,7649,7650,7651,7652,7653,7654,7655,7656,7657,7658,7659,7660,7661,7662,7663, -7664,7665,7666,7667,7668,7669,7670,7671,7672,7673,7674,7675,7676,7677,7678,7679, -7680,7681,7682,7683,7684,7685,7686,7687,7688,7689,7690,7691,7692,7693,7694,7695, -7696,7697,7698,7699,7700,7701,7702,7703,7704,7705,7706,7707,7708,7709,7710,7711, -7712,7713,7714,7715,7716,7717,7718,7719,7720,7721,7722,7723,7724,7725,7726,7727, -7728,7729,7730,7731,7732,7733,7734,7735,7736,7737,7738,7739,7740,7741,7742,7743, -7744,7745,7746,7747,7748,7749,7750,7751,7752,7753,7754,7755,7756,7757,7758,7759, -7760,7761,7762,7763,7764,7765,7766,7767,7768,7769,7770,7771,7772,7773,7774,7775, -7776,7777,7778,7779,7780,7781,7782,7783,7784,7785,7786,7787,7788,7789,7790,7791, -7792,7793,7794,7795,7796,7797,7798,7799,7800,7801,7802,7803,7804,7805,7806,7807, -7808,7809,7810,7811,7812,7813,7814,7815,7816,7817,7818,7819,7820,7821,7822,7823, -7824,7825,7826,7827,7828,7829,7830,7831,7832,7833,7834,7835,7836,7837,7838,7839, -7840,7841,7842,7843,7844,7845,7846,7847,7848,7849,7850,7851,7852,7853,7854,7855, -7856,7857,7858,7859,7860,7861,7862,7863,7864,7865,7866,7867,7868,7869,7870,7871, -7872,7873,7874,7875,7876,7877,7878,7879,7880,7881,7882,7883,7884,7885,7886,7887, -7888,7889,7890,7891,7892,7893,7894,7895,7896,7897,7898,7899,7900,7901,7902,7903, -7904,7905,7906,7907,7908,7909,7910,7911,7912,7913,7914,7915,7916,7917,7918,7919, -7920,7921,7922,7923,7924,7925,7926,7927,7928,7929,7930,7931,7932,7933,7934,7935, -7936,7937,7938,7939,7940,7941,7942,7943,7944,7945,7946,7947,7948,7949,7950,7951, -7952,7953,7954,7955,7956,7957,7958,7959,7960,7961,7962,7963,7964,7965,7966,7967, -7968,7969,7970,7971,7972,7973,7974,7975,7976,7977,7978,7979,7980,7981,7982,7983, -7984,7985,7986,7987,7988,7989,7990,7991,7992,7993,7994,7995,7996,7997,7998,7999, -8000,8001,8002,8003,8004,8005,8006,8007,8008,8009,8010,8011,8012,8013,8014,8015, -8016,8017,8018,8019,8020,8021,8022,8023,8024,8025,8026,8027,8028,8029,8030,8031, -8032,8033,8034,8035,8036,8037,8038,8039,8040,8041,8042,8043,8044,8045,8046,8047, -8048,8049,8050,8051,8052,8053,8054,8055,8056,8057,8058,8059,8060,8061,8062,8063, -8064,8065,8066,8067,8068,8069,8070,8071,8072,8073,8074,8075,8076,8077,8078,8079, -8080,8081,8082,8083,8084,8085,8086,8087,8088,8089,8090,8091,8092,8093,8094,8095, -8096,8097,8098,8099,8100,8101,8102,8103,8104,8105,8106,8107,8108,8109,8110,8111, -8112,8113,8114,8115,8116,8117,8118,8119,8120,8121,8122,8123,8124,8125,8126,8127, -8128,8129,8130,8131,8132,8133,8134,8135,8136,8137,8138,8139,8140,8141,8142,8143, -8144,8145,8146,8147,8148,8149,8150,8151,8152,8153,8154,8155,8156,8157,8158,8159, -8160,8161,8162,8163,8164,8165,8166,8167,8168,8169,8170,8171,8172,8173,8174,8175, -8176,8177,8178,8179,8180,8181,8182,8183,8184,8185,8186,8187,8188,8189,8190,8191, -8192,8193,8194,8195,8196,8197,8198,8199,8200,8201,8202,8203,8204,8205,8206,8207, -8208,8209,8210,8211,8212,8213,8214,8215,8216,8217,8218,8219,8220,8221,8222,8223, -8224,8225,8226,8227,8228,8229,8230,8231,8232,8233,8234,8235,8236,8237,8238,8239, -8240,8241,8242,8243,8244,8245,8246,8247,8248,8249,8250,8251,8252,8253,8254,8255, -8256,8257,8258,8259,8260,8261,8262,8263,8264,8265,8266,8267,8268,8269,8270,8271, -8272,8273,8274,8275,8276,8277,8278,8279,8280,8281,8282,8283,8284,8285,8286,8287, -8288,8289,8290,8291,8292,8293,8294,8295,8296,8297,8298,8299,8300,8301,8302,8303, -8304,8305,8306,8307,8308,8309,8310,8311,8312,8313,8314,8315,8316,8317,8318,8319, -8320,8321,8322,8323,8324,8325,8326,8327,8328,8329,8330,8331,8332,8333,8334,8335, -8336,8337,8338,8339,8340,8341,8342,8343,8344,8345,8346,8347,8348,8349,8350,8351, -8352,8353,8354,8355,8356,8357,8358,8359,8360,8361,8362,8363,8364,8365,8366,8367, -8368,8369,8370,8371,8372,8373,8374,8375,8376,8377,8378,8379,8380,8381,8382,8383, -8384,8385,8386,8387,8388,8389,8390,8391,8392,8393,8394,8395,8396,8397,8398,8399, -8400,8401,8402,8403,8404,8405,8406,8407,8408,8409,8410,8411,8412,8413,8414,8415, -8416,8417,8418,8419,8420,8421,8422,8423,8424,8425,8426,8427,8428,8429,8430,8431, -8432,8433,8434,8435,8436,8437,8438,8439,8440,8441,8442,8443,8444,8445,8446,8447, -8448,8449,8450,8451,8452,8453,8454,8455,8456,8457,8458,8459,8460,8461,8462,8463, -8464,8465,8466,8467,8468,8469,8470,8471,8472,8473,8474,8475,8476,8477,8478,8479, -8480,8481,8482,8483,8484,8485,8486,8487,8488,8489,8490,8491,8492,8493,8494,8495, -8496,8497,8498,8499,8500,8501,8502,8503,8504,8505,8506,8507,8508,8509,8510,8511, -8512,8513,8514,8515,8516,8517,8518,8519,8520,8521,8522,8523,8524,8525,8526,8527, -8528,8529,8530,8531,8532,8533,8534,8535,8536,8537,8538,8539,8540,8541,8542,8543, -8544,8545,8546,8547,8548,8549,8550,8551,8552,8553,8554,8555,8556,8557,8558,8559, -8560,8561,8562,8563,8564,8565,8566,8567,8568,8569,8570,8571,8572,8573,8574,8575, -8576,8577,8578,8579,8580,8581,8582,8583,8584,8585,8586,8587,8588,8589,8590,8591, -8592,8593,8594,8595,8596,8597,8598,8599,8600,8601,8602,8603,8604,8605,8606,8607, -8608,8609,8610,8611,8612,8613,8614,8615,8616,8617,8618,8619,8620,8621,8622,8623, -8624,8625,8626,8627,8628,8629,8630,8631,8632,8633,8634,8635,8636,8637,8638,8639, -8640,8641,8642,8643,8644,8645,8646,8647,8648,8649,8650,8651,8652,8653,8654,8655, -8656,8657,8658,8659,8660,8661,8662,8663,8664,8665,8666,8667,8668,8669,8670,8671, -8672,8673,8674,8675,8676,8677,8678,8679,8680,8681,8682,8683,8684,8685,8686,8687, -8688,8689,8690,8691,8692,8693,8694,8695,8696,8697,8698,8699,8700,8701,8702,8703, -8704,8705,8706,8707,8708,8709,8710,8711,8712,8713,8714,8715,8716,8717,8718,8719, -8720,8721,8722,8723,8724,8725,8726,8727,8728,8729,8730,8731,8732,8733,8734,8735, -8736,8737,8738,8739,8740,8741 -****************************************************************************************/ -}; - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/EUCTWFreq.tab b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/EUCTWFreq.tab deleted file mode 100644 index 3a189ae..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/EUCTWFreq.tab +++ /dev/null @@ -1,447 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -// EUCTW frequency table -// Converted from big5 work -// by Taiwan's Mandarin Promotion Council -// - - -/****************************************************************************** - * 128 --> 0.42261 - * 256 --> 0.57851 - * 512 --> 0.74851 - * 1024 --> 0.89384 - * 2048 --> 0.97583 - * - * Idea Distribution Ratio = 0.74851/(1-0.74851) =2.98 - * Random Distribution Ration = 512/(5401-512)=0.105 - * - * Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR - *****************************************************************************/ - -#define EUCTW_TYPICAL_DISTRIBUTION_RATIO (float)0.75 - -//Char to FreqOrder table , -#define EUCTW_TABLE_SIZE 8102 - -static const PRInt16 EUCTWCharToFreqOrder[] = -{ - 1,1800,1506, 255,1431, 198, 9, 82, 6,7310, 177, 202,3615,1256,2808, 110, // 2742 -3735, 33,3241, 261, 76, 44,2113, 16,2931,2184,1176, 659,3868, 26,3404,2643, // 2758 -1198,3869,3313,4060, 410,2211, 302, 590, 361,1963, 8, 204, 58,4296,7311,1931, // 2774 - 63,7312,7313, 317,1614, 75, 222, 159,4061,2412,1480,7314,3500,3068, 224,2809, // 2790 -3616, 3, 10,3870,1471, 29,2774,1135,2852,1939, 873, 130,3242,1123, 312,7315, // 2806 -4297,2051, 507, 252, 682,7316, 142,1914, 124, 206,2932, 34,3501,3173, 64, 604, // 2822 -7317,2494,1976,1977, 155,1990, 645, 641,1606,7318,3405, 337, 72, 406,7319, 80, // 2838 - 630, 238,3174,1509, 263, 939,1092,2644, 756,1440,1094,3406, 449, 69,2969, 591, // 2854 - 179,2095, 471, 115,2034,1843, 60, 50,2970, 134, 806,1868, 734,2035,3407, 180, // 2870 - 995,1607, 156, 537,2893, 688,7320, 319,1305, 779,2144, 514,2374, 298,4298, 359, // 2886 -2495, 90,2707,1338, 663, 11, 906,1099,2545, 20,2436, 182, 532,1716,7321, 732, // 2902 -1376,4062,1311,1420,3175, 25,2312,1056, 113, 399, 382,1949, 242,3408,2467, 529, // 2918 -3243, 475,1447,3617,7322, 117, 21, 656, 810,1297,2295,2329,3502,7323, 126,4063, // 2934 - 706, 456, 150, 613,4299, 71,1118,2036,4064, 145,3069, 85, 835, 486,2114,1246, // 2950 -1426, 428, 727,1285,1015, 800, 106, 623, 303,1281,7324,2127,2354, 347,3736, 221, // 2966 -3503,3110,7325,1955,1153,4065, 83, 296,1199,3070, 192, 624, 93,7326, 822,1897, // 2982 -2810,3111, 795,2064, 991,1554,1542,1592, 27, 43,2853, 859, 139,1456, 860,4300, // 2998 - 437, 712,3871, 164,2392,3112, 695, 211,3017,2096, 195,3872,1608,3504,3505,3618, // 3014 -3873, 234, 811,2971,2097,3874,2229,1441,3506,1615,2375, 668,2076,1638, 305, 228, // 3030 -1664,4301, 467, 415,7327, 262,2098,1593, 239, 108, 300, 200,1033, 512,1247,2077, // 3046 -7328,7329,2173,3176,3619,2673, 593, 845,1062,3244, 88,1723,2037,3875,1950, 212, // 3062 - 266, 152, 149, 468,1898,4066,4302, 77, 187,7330,3018, 37, 5,2972,7331,3876, // 3078 -7332,7333, 39,2517,4303,2894,3177,2078, 55, 148, 74,4304, 545, 483,1474,1029, // 3094 -1665, 217,1869,1531,3113,1104,2645,4067, 24, 172,3507, 900,3877,3508,3509,4305, // 3110 - 32,1408,2811,1312, 329, 487,2355,2247,2708, 784,2674, 4,3019,3314,1427,1788, // 3126 - 188, 109, 499,7334,3620,1717,1789, 888,1217,3020,4306,7335,3510,7336,3315,1520, // 3142 -3621,3878, 196,1034, 775,7337,7338, 929,1815, 249, 439, 38,7339,1063,7340, 794, // 3158 -3879,1435,2296, 46, 178,3245,2065,7341,2376,7342, 214,1709,4307, 804, 35, 707, // 3174 - 324,3622,1601,2546, 140, 459,4068,7343,7344,1365, 839, 272, 978,2257,2572,3409, // 3190 -2128,1363,3623,1423, 697, 100,3071, 48, 70,1231, 495,3114,2193,7345,1294,7346, // 3206 -2079, 462, 586,1042,3246, 853, 256, 988, 185,2377,3410,1698, 434,1084,7347,3411, // 3222 - 314,2615,2775,4308,2330,2331, 569,2280, 637,1816,2518, 757,1162,1878,1616,3412, // 3238 - 287,1577,2115, 768,4309,1671,2854,3511,2519,1321,3737, 909,2413,7348,4069, 933, // 3254 -3738,7349,2052,2356,1222,4310, 765,2414,1322, 786,4311,7350,1919,1462,1677,2895, // 3270 -1699,7351,4312,1424,2437,3115,3624,2590,3316,1774,1940,3413,3880,4070, 309,1369, // 3286 -1130,2812, 364,2230,1653,1299,3881,3512,3882,3883,2646, 525,1085,3021, 902,2000, // 3302 -1475, 964,4313, 421,1844,1415,1057,2281, 940,1364,3116, 376,4314,4315,1381, 7, // 3318 -2520, 983,2378, 336,1710,2675,1845, 321,3414, 559,1131,3022,2742,1808,1132,1313, // 3334 - 265,1481,1857,7352, 352,1203,2813,3247, 167,1089, 420,2814, 776, 792,1724,3513, // 3350 -4071,2438,3248,7353,4072,7354, 446, 229, 333,2743, 901,3739,1200,1557,4316,2647, // 3366 -1920, 395,2744,2676,3740,4073,1835, 125, 916,3178,2616,4317,7355,7356,3741,7357, // 3382 -7358,7359,4318,3117,3625,1133,2547,1757,3415,1510,2313,1409,3514,7360,2145, 438, // 3398 -2591,2896,2379,3317,1068, 958,3023, 461, 311,2855,2677,4074,1915,3179,4075,1978, // 3414 - 383, 750,2745,2617,4076, 274, 539, 385,1278,1442,7361,1154,1964, 384, 561, 210, // 3430 - 98,1295,2548,3515,7362,1711,2415,1482,3416,3884,2897,1257, 129,7363,3742, 642, // 3446 - 523,2776,2777,2648,7364, 141,2231,1333, 68, 176, 441, 876, 907,4077, 603,2592, // 3462 - 710, 171,3417, 404, 549, 18,3118,2393,1410,3626,1666,7365,3516,4319,2898,4320, // 3478 -7366,2973, 368,7367, 146, 366, 99, 871,3627,1543, 748, 807,1586,1185, 22,2258, // 3494 - 379,3743,3180,7368,3181, 505,1941,2618,1991,1382,2314,7369, 380,2357, 218, 702, // 3510 -1817,1248,3418,3024,3517,3318,3249,7370,2974,3628, 930,3250,3744,7371, 59,7372, // 3526 - 585, 601,4078, 497,3419,1112,1314,4321,1801,7373,1223,1472,2174,7374, 749,1836, // 3542 - 690,1899,3745,1772,3885,1476, 429,1043,1790,2232,2116, 917,4079, 447,1086,1629, // 3558 -7375, 556,7376,7377,2020,1654, 844,1090, 105, 550, 966,1758,2815,1008,1782, 686, // 3574 -1095,7378,2282, 793,1602,7379,3518,2593,4322,4080,2933,2297,4323,3746, 980,2496, // 3590 - 544, 353, 527,4324, 908,2678,2899,7380, 381,2619,1942,1348,7381,1341,1252, 560, // 3606 -3072,7382,3420,2856,7383,2053, 973, 886,2080, 143,4325,7384,7385, 157,3886, 496, // 3622 -4081, 57, 840, 540,2038,4326,4327,3421,2117,1445, 970,2259,1748,1965,2081,4082, // 3638 -3119,1234,1775,3251,2816,3629, 773,1206,2129,1066,2039,1326,3887,1738,1725,4083, // 3654 - 279,3120, 51,1544,2594, 423,1578,2130,2066, 173,4328,1879,7386,7387,1583, 264, // 3670 - 610,3630,4329,2439, 280, 154,7388,7389,7390,1739, 338,1282,3073, 693,2857,1411, // 3686 -1074,3747,2440,7391,4330,7392,7393,1240, 952,2394,7394,2900,1538,2679, 685,1483, // 3702 -4084,2468,1436, 953,4085,2054,4331, 671,2395, 79,4086,2441,3252, 608, 567,2680, // 3718 -3422,4087,4088,1691, 393,1261,1791,2396,7395,4332,7396,7397,7398,7399,1383,1672, // 3734 -3748,3182,1464, 522,1119, 661,1150, 216, 675,4333,3888,1432,3519, 609,4334,2681, // 3750 -2397,7400,7401,7402,4089,3025, 0,7403,2469, 315, 231,2442, 301,3319,4335,2380, // 3766 -7404, 233,4090,3631,1818,4336,4337,7405, 96,1776,1315,2082,7406, 257,7407,1809, // 3782 -3632,2709,1139,1819,4091,2021,1124,2163,2778,1777,2649,7408,3074, 363,1655,3183, // 3798 -7409,2975,7410,7411,7412,3889,1567,3890, 718, 103,3184, 849,1443, 341,3320,2934, // 3814 -1484,7413,1712, 127, 67, 339,4092,2398, 679,1412, 821,7414,7415, 834, 738, 351, // 3830 -2976,2146, 846, 235,1497,1880, 418,1992,3749,2710, 186,1100,2147,2746,3520,1545, // 3846 -1355,2935,2858,1377, 583,3891,4093,2573,2977,7416,1298,3633,1078,2549,3634,2358, // 3862 - 78,3750,3751, 267,1289,2099,2001,1594,4094, 348, 369,1274,2194,2175,1837,4338, // 3878 -1820,2817,3635,2747,2283,2002,4339,2936,2748, 144,3321, 882,4340,3892,2749,3423, // 3894 -4341,2901,7417,4095,1726, 320,7418,3893,3026, 788,2978,7419,2818,1773,1327,2859, // 3910 -3894,2819,7420,1306,4342,2003,1700,3752,3521,2359,2650, 787,2022, 506, 824,3636, // 3926 - 534, 323,4343,1044,3322,2023,1900, 946,3424,7421,1778,1500,1678,7422,1881,4344, // 3942 - 165, 243,4345,3637,2521, 123, 683,4096, 764,4346, 36,3895,1792, 589,2902, 816, // 3958 - 626,1667,3027,2233,1639,1555,1622,3753,3896,7423,3897,2860,1370,1228,1932, 891, // 3974 -2083,2903, 304,4097,7424, 292,2979,2711,3522, 691,2100,4098,1115,4347, 118, 662, // 3990 -7425, 611,1156, 854,2381,1316,2861, 2, 386, 515,2904,7426,7427,3253, 868,2234, // 4006 -1486, 855,2651, 785,2212,3028,7428,1040,3185,3523,7429,3121, 448,7430,1525,7431, // 4022 -2164,4348,7432,3754,7433,4099,2820,3524,3122, 503, 818,3898,3123,1568, 814, 676, // 4038 -1444, 306,1749,7434,3755,1416,1030, 197,1428, 805,2821,1501,4349,7435,7436,7437, // 4054 -1993,7438,4350,7439,7440,2195, 13,2779,3638,2980,3124,1229,1916,7441,3756,2131, // 4070 -7442,4100,4351,2399,3525,7443,2213,1511,1727,1120,7444,7445, 646,3757,2443, 307, // 4086 -7446,7447,1595,3186,7448,7449,7450,3639,1113,1356,3899,1465,2522,2523,7451, 519, // 4102 -7452, 128,2132, 92,2284,1979,7453,3900,1512, 342,3125,2196,7454,2780,2214,1980, // 4118 -3323,7455, 290,1656,1317, 789, 827,2360,7456,3758,4352, 562, 581,3901,7457, 401, // 4134 -4353,2248, 94,4354,1399,2781,7458,1463,2024,4355,3187,1943,7459, 828,1105,4101, // 4150 -1262,1394,7460,4102, 605,4356,7461,1783,2862,7462,2822, 819,2101, 578,2197,2937, // 4166 -7463,1502, 436,3254,4103,3255,2823,3902,2905,3425,3426,7464,2712,2315,7465,7466, // 4182 -2332,2067, 23,4357, 193, 826,3759,2102, 699,1630,4104,3075, 390,1793,1064,3526, // 4198 -7467,1579,3076,3077,1400,7468,4105,1838,1640,2863,7469,4358,4359, 137,4106, 598, // 4214 -3078,1966, 780, 104, 974,2938,7470, 278, 899, 253, 402, 572, 504, 493,1339,7471, // 4230 -3903,1275,4360,2574,2550,7472,3640,3029,3079,2249, 565,1334,2713, 863, 41,7473, // 4246 -7474,4361,7475,1657,2333, 19, 463,2750,4107, 606,7476,2981,3256,1087,2084,1323, // 4262 -2652,2982,7477,1631,1623,1750,4108,2682,7478,2864, 791,2714,2653,2334, 232,2416, // 4278 -7479,2983,1498,7480,2654,2620, 755,1366,3641,3257,3126,2025,1609, 119,1917,3427, // 4294 - 862,1026,4109,7481,3904,3760,4362,3905,4363,2260,1951,2470,7482,1125, 817,4110, // 4310 -4111,3906,1513,1766,2040,1487,4112,3030,3258,2824,3761,3127,7483,7484,1507,7485, // 4326 -2683, 733, 40,1632,1106,2865, 345,4113, 841,2524, 230,4364,2984,1846,3259,3428, // 4342 -7486,1263, 986,3429,7487, 735, 879, 254,1137, 857, 622,1300,1180,1388,1562,3907, // 4358 -3908,2939, 967,2751,2655,1349, 592,2133,1692,3324,2985,1994,4114,1679,3909,1901, // 4374 -2185,7488, 739,3642,2715,1296,1290,7489,4115,2198,2199,1921,1563,2595,2551,1870, // 4390 -2752,2986,7490, 435,7491, 343,1108, 596, 17,1751,4365,2235,3430,3643,7492,4366, // 4406 - 294,3527,2940,1693, 477, 979, 281,2041,3528, 643,2042,3644,2621,2782,2261,1031, // 4422 -2335,2134,2298,3529,4367, 367,1249,2552,7493,3530,7494,4368,1283,3325,2004, 240, // 4438 -1762,3326,4369,4370, 836,1069,3128, 474,7495,2148,2525, 268,3531,7496,3188,1521, // 4454 -1284,7497,1658,1546,4116,7498,3532,3533,7499,4117,3327,2684,1685,4118, 961,1673, // 4470 -2622, 190,2005,2200,3762,4371,4372,7500, 570,2497,3645,1490,7501,4373,2623,3260, // 4486 -1956,4374, 584,1514, 396,1045,1944,7502,4375,1967,2444,7503,7504,4376,3910, 619, // 4502 -7505,3129,3261, 215,2006,2783,2553,3189,4377,3190,4378, 763,4119,3763,4379,7506, // 4518 -7507,1957,1767,2941,3328,3646,1174, 452,1477,4380,3329,3130,7508,2825,1253,2382, // 4534 -2186,1091,2285,4120, 492,7509, 638,1169,1824,2135,1752,3911, 648, 926,1021,1324, // 4550 -4381, 520,4382, 997, 847,1007, 892,4383,3764,2262,1871,3647,7510,2400,1784,4384, // 4566 -1952,2942,3080,3191,1728,4121,2043,3648,4385,2007,1701,3131,1551, 30,2263,4122, // 4582 -7511,2026,4386,3534,7512, 501,7513,4123, 594,3431,2165,1821,3535,3432,3536,3192, // 4598 - 829,2826,4124,7514,1680,3132,1225,4125,7515,3262,4387,4126,3133,2336,7516,4388, // 4614 -4127,7517,3912,3913,7518,1847,2383,2596,3330,7519,4389, 374,3914, 652,4128,4129, // 4630 - 375,1140, 798,7520,7521,7522,2361,4390,2264, 546,1659, 138,3031,2445,4391,7523, // 4646 -2250, 612,1848, 910, 796,3765,1740,1371, 825,3766,3767,7524,2906,2554,7525, 692, // 4662 - 444,3032,2624, 801,4392,4130,7526,1491, 244,1053,3033,4131,4132, 340,7527,3915, // 4678 -1041,2987, 293,1168, 87,1357,7528,1539, 959,7529,2236, 721, 694,4133,3768, 219, // 4694 -1478, 644,1417,3331,2656,1413,1401,1335,1389,3916,7530,7531,2988,2362,3134,1825, // 4710 - 730,1515, 184,2827, 66,4393,7532,1660,2943, 246,3332, 378,1457, 226,3433, 975, // 4726 -3917,2944,1264,3537, 674, 696,7533, 163,7534,1141,2417,2166, 713,3538,3333,4394, // 4742 -3918,7535,7536,1186, 15,7537,1079,1070,7538,1522,3193,3539, 276,1050,2716, 758, // 4758 -1126, 653,2945,3263,7539,2337, 889,3540,3919,3081,2989, 903,1250,4395,3920,3434, // 4774 -3541,1342,1681,1718, 766,3264, 286, 89,2946,3649,7540,1713,7541,2597,3334,2990, // 4790 -7542,2947,2215,3194,2866,7543,4396,2498,2526, 181, 387,1075,3921, 731,2187,3335, // 4806 -7544,3265, 310, 313,3435,2299, 770,4134, 54,3034, 189,4397,3082,3769,3922,7545, // 4822 -1230,1617,1849, 355,3542,4135,4398,3336, 111,4136,3650,1350,3135,3436,3035,4137, // 4838 -2149,3266,3543,7546,2784,3923,3924,2991, 722,2008,7547,1071, 247,1207,2338,2471, // 4854 -1378,4399,2009, 864,1437,1214,4400, 373,3770,1142,2216, 667,4401, 442,2753,2555, // 4870 -3771,3925,1968,4138,3267,1839, 837, 170,1107, 934,1336,1882,7548,7549,2118,4139, // 4886 -2828, 743,1569,7550,4402,4140, 582,2384,1418,3437,7551,1802,7552, 357,1395,1729, // 4902 -3651,3268,2418,1564,2237,7553,3083,3772,1633,4403,1114,2085,4141,1532,7554, 482, // 4918 -2446,4404,7555,7556,1492, 833,1466,7557,2717,3544,1641,2829,7558,1526,1272,3652, // 4934 -4142,1686,1794, 416,2556,1902,1953,1803,7559,3773,2785,3774,1159,2316,7560,2867, // 4950 -4405,1610,1584,3036,2419,2754, 443,3269,1163,3136,7561,7562,3926,7563,4143,2499, // 4966 -3037,4406,3927,3137,2103,1647,3545,2010,1872,4144,7564,4145, 431,3438,7565, 250, // 4982 - 97, 81,4146,7566,1648,1850,1558, 160, 848,7567, 866, 740,1694,7568,2201,2830, // 4998 -3195,4147,4407,3653,1687, 950,2472, 426, 469,3196,3654,3655,3928,7569,7570,1188, // 5014 - 424,1995, 861,3546,4148,3775,2202,2685, 168,1235,3547,4149,7571,2086,1674,4408, // 5030 -3337,3270, 220,2557,1009,7572,3776, 670,2992, 332,1208, 717,7573,7574,3548,2447, // 5046 -3929,3338,7575, 513,7576,1209,2868,3339,3138,4409,1080,7577,7578,7579,7580,2527, // 5062 -3656,3549, 815,1587,3930,3931,7581,3550,3439,3777,1254,4410,1328,3038,1390,3932, // 5078 -1741,3933,3778,3934,7582, 236,3779,2448,3271,7583,7584,3657,3780,1273,3781,4411, // 5094 -7585, 308,7586,4412, 245,4413,1851,2473,1307,2575, 430, 715,2136,2449,7587, 270, // 5110 - 199,2869,3935,7588,3551,2718,1753, 761,1754, 725,1661,1840,4414,3440,3658,7589, // 5126 -7590, 587, 14,3272, 227,2598, 326, 480,2265, 943,2755,3552, 291, 650,1883,7591, // 5142 -1702,1226, 102,1547, 62,3441, 904,4415,3442,1164,4150,7592,7593,1224,1548,2756, // 5158 - 391, 498,1493,7594,1386,1419,7595,2055,1177,4416, 813, 880,1081,2363, 566,1145, // 5174 -4417,2286,1001,1035,2558,2599,2238, 394,1286,7596,7597,2068,7598, 86,1494,1730, // 5190 -3936, 491,1588, 745, 897,2948, 843,3340,3937,2757,2870,3273,1768, 998,2217,2069, // 5206 - 397,1826,1195,1969,3659,2993,3341, 284,7599,3782,2500,2137,2119,1903,7600,3938, // 5222 -2150,3939,4151,1036,3443,1904, 114,2559,4152, 209,1527,7601,7602,2949,2831,2625, // 5238 -2385,2719,3139, 812,2560,7603,3274,7604,1559, 737,1884,3660,1210, 885, 28,2686, // 5254 -3553,3783,7605,4153,1004,1779,4418,7606, 346,1981,2218,2687,4419,3784,1742, 797, // 5270 -1642,3940,1933,1072,1384,2151, 896,3941,3275,3661,3197,2871,3554,7607,2561,1958, // 5286 -4420,2450,1785,7608,7609,7610,3942,4154,1005,1308,3662,4155,2720,4421,4422,1528, // 5302 -2600, 161,1178,4156,1982, 987,4423,1101,4157, 631,3943,1157,3198,2420,1343,1241, // 5318 -1016,2239,2562, 372, 877,2339,2501,1160, 555,1934, 911,3944,7611, 466,1170, 169, // 5334 -1051,2907,2688,3663,2474,2994,1182,2011,2563,1251,2626,7612, 992,2340,3444,1540, // 5350 -2721,1201,2070,2401,1996,2475,7613,4424, 528,1922,2188,1503,1873,1570,2364,3342, // 5366 -3276,7614, 557,1073,7615,1827,3445,2087,2266,3140,3039,3084, 767,3085,2786,4425, // 5382 -1006,4158,4426,2341,1267,2176,3664,3199, 778,3945,3200,2722,1597,2657,7616,4427, // 5398 -7617,3446,7618,7619,7620,3277,2689,1433,3278, 131, 95,1504,3946, 723,4159,3141, // 5414 -1841,3555,2758,2189,3947,2027,2104,3665,7621,2995,3948,1218,7622,3343,3201,3949, // 5430 -4160,2576, 248,1634,3785, 912,7623,2832,3666,3040,3786, 654, 53,7624,2996,7625, // 5446 -1688,4428, 777,3447,1032,3950,1425,7626, 191, 820,2120,2833, 971,4429, 931,3202, // 5462 - 135, 664, 783,3787,1997, 772,2908,1935,3951,3788,4430,2909,3203, 282,2723, 640, // 5478 -1372,3448,1127, 922, 325,3344,7627,7628, 711,2044,7629,7630,3952,2219,2787,1936, // 5494 -3953,3345,2220,2251,3789,2300,7631,4431,3790,1258,3279,3954,3204,2138,2950,3955, // 5510 -3956,7632,2221, 258,3205,4432, 101,1227,7633,3280,1755,7634,1391,3281,7635,2910, // 5526 -2056, 893,7636,7637,7638,1402,4161,2342,7639,7640,3206,3556,7641,7642, 878,1325, // 5542 -1780,2788,4433, 259,1385,2577, 744,1183,2267,4434,7643,3957,2502,7644, 684,1024, // 5558 -4162,7645, 472,3557,3449,1165,3282,3958,3959, 322,2152, 881, 455,1695,1152,1340, // 5574 - 660, 554,2153,4435,1058,4436,4163, 830,1065,3346,3960,4437,1923,7646,1703,1918, // 5590 -7647, 932,2268, 122,7648,4438, 947, 677,7649,3791,2627, 297,1905,1924,2269,4439, // 5606 -2317,3283,7650,7651,4164,7652,4165, 84,4166, 112, 989,7653, 547,1059,3961, 701, // 5622 -3558,1019,7654,4167,7655,3450, 942, 639, 457,2301,2451, 993,2951, 407, 851, 494, // 5638 -4440,3347, 927,7656,1237,7657,2421,3348, 573,4168, 680, 921,2911,1279,1874, 285, // 5654 - 790,1448,1983, 719,2167,7658,7659,4441,3962,3963,1649,7660,1541, 563,7661,1077, // 5670 -7662,3349,3041,3451, 511,2997,3964,3965,3667,3966,1268,2564,3350,3207,4442,4443, // 5686 -7663, 535,1048,1276,1189,2912,2028,3142,1438,1373,2834,2952,1134,2012,7664,4169, // 5702 -1238,2578,3086,1259,7665, 700,7666,2953,3143,3668,4170,7667,4171,1146,1875,1906, // 5718 -4444,2601,3967, 781,2422, 132,1589, 203, 147, 273,2789,2402, 898,1786,2154,3968, // 5734 -3969,7668,3792,2790,7669,7670,4445,4446,7671,3208,7672,1635,3793, 965,7673,1804, // 5750 -2690,1516,3559,1121,1082,1329,3284,3970,1449,3794, 65,1128,2835,2913,2759,1590, // 5766 -3795,7674,7675, 12,2658, 45, 976,2579,3144,4447, 517,2528,1013,1037,3209,7676, // 5782 -3796,2836,7677,3797,7678,3452,7679,2602, 614,1998,2318,3798,3087,2724,2628,7680, // 5798 -2580,4172, 599,1269,7681,1810,3669,7682,2691,3088, 759,1060, 489,1805,3351,3285, // 5814 -1358,7683,7684,2386,1387,1215,2629,2252, 490,7685,7686,4173,1759,2387,2343,7687, // 5830 -4448,3799,1907,3971,2630,1806,3210,4449,3453,3286,2760,2344, 874,7688,7689,3454, // 5846 -3670,1858, 91,2914,3671,3042,3800,4450,7690,3145,3972,2659,7691,3455,1202,1403, // 5862 -3801,2954,2529,1517,2503,4451,3456,2504,7692,4452,7693,2692,1885,1495,1731,3973, // 5878 -2365,4453,7694,2029,7695,7696,3974,2693,1216, 237,2581,4174,2319,3975,3802,4454, // 5894 -4455,2694,3560,3457, 445,4456,7697,7698,7699,7700,2761, 61,3976,3672,1822,3977, // 5910 -7701, 687,2045, 935, 925, 405,2660, 703,1096,1859,2725,4457,3978,1876,1367,2695, // 5926 -3352, 918,2105,1781,2476, 334,3287,1611,1093,4458, 564,3146,3458,3673,3353, 945, // 5942 -2631,2057,4459,7702,1925, 872,4175,7703,3459,2696,3089, 349,4176,3674,3979,4460, // 5958 -3803,4177,3675,2155,3980,4461,4462,4178,4463,2403,2046, 782,3981, 400, 251,4179, // 5974 -1624,7704,7705, 277,3676, 299,1265, 476,1191,3804,2121,4180,4181,1109, 205,7706, // 5990 -2582,1000,2156,3561,1860,7707,7708,7709,4464,7710,4465,2565, 107,2477,2157,3982, // 6006 -3460,3147,7711,1533, 541,1301, 158, 753,4182,2872,3562,7712,1696, 370,1088,4183, // 6022 -4466,3563, 579, 327, 440, 162,2240, 269,1937,1374,3461, 968,3043, 56,1396,3090, // 6038 -2106,3288,3354,7713,1926,2158,4467,2998,7714,3564,7715,7716,3677,4468,2478,7717, // 6054 -2791,7718,1650,4469,7719,2603,7720,7721,3983,2661,3355,1149,3356,3984,3805,3985, // 6070 -7722,1076, 49,7723, 951,3211,3289,3290, 450,2837, 920,7724,1811,2792,2366,4184, // 6086 -1908,1138,2367,3806,3462,7725,3212,4470,1909,1147,1518,2423,4471,3807,7726,4472, // 6102 -2388,2604, 260,1795,3213,7727,7728,3808,3291, 708,7729,3565,1704,7730,3566,1351, // 6118 -1618,3357,2999,1886, 944,4185,3358,4186,3044,3359,4187,7731,3678, 422, 413,1714, // 6134 -3292, 500,2058,2345,4188,2479,7732,1344,1910, 954,7733,1668,7734,7735,3986,2404, // 6150 -4189,3567,3809,4190,7736,2302,1318,2505,3091, 133,3092,2873,4473, 629, 31,2838, // 6166 -2697,3810,4474, 850, 949,4475,3987,2955,1732,2088,4191,1496,1852,7737,3988, 620, // 6182 -3214, 981,1242,3679,3360,1619,3680,1643,3293,2139,2452,1970,1719,3463,2168,7738, // 6198 -3215,7739,7740,3361,1828,7741,1277,4476,1565,2047,7742,1636,3568,3093,7743, 869, // 6214 -2839, 655,3811,3812,3094,3989,3000,3813,1310,3569,4477,7744,7745,7746,1733, 558, // 6230 -4478,3681, 335,1549,3045,1756,4192,3682,1945,3464,1829,1291,1192, 470,2726,2107, // 6246 -2793, 913,1054,3990,7747,1027,7748,3046,3991,4479, 982,2662,3362,3148,3465,3216, // 6262 -3217,1946,2794,7749, 571,4480,7750,1830,7751,3570,2583,1523,2424,7752,2089, 984, // 6278 -4481,3683,1959,7753,3684, 852, 923,2795,3466,3685, 969,1519, 999,2048,2320,1705, // 6294 -7754,3095, 615,1662, 151, 597,3992,2405,2321,1049, 275,4482,3686,4193, 568,3687, // 6310 -3571,2480,4194,3688,7755,2425,2270, 409,3218,7756,1566,2874,3467,1002, 769,2840, // 6326 - 194,2090,3149,3689,2222,3294,4195, 628,1505,7757,7758,1763,2177,3001,3993, 521, // 6342 -1161,2584,1787,2203,2406,4483,3994,1625,4196,4197, 412, 42,3096, 464,7759,2632, // 6358 -4484,3363,1760,1571,2875,3468,2530,1219,2204,3814,2633,2140,2368,4485,4486,3295, // 6374 -1651,3364,3572,7760,7761,3573,2481,3469,7762,3690,7763,7764,2271,2091, 460,7765, // 6390 -4487,7766,3002, 962, 588,3574, 289,3219,2634,1116, 52,7767,3047,1796,7768,7769, // 6406 -7770,1467,7771,1598,1143,3691,4198,1984,1734,1067,4488,1280,3365, 465,4489,1572, // 6422 - 510,7772,1927,2241,1812,1644,3575,7773,4490,3692,7774,7775,2663,1573,1534,7776, // 6438 -7777,4199, 536,1807,1761,3470,3815,3150,2635,7778,7779,7780,4491,3471,2915,1911, // 6454 -2796,7781,3296,1122, 377,3220,7782, 360,7783,7784,4200,1529, 551,7785,2059,3693, // 6470 -1769,2426,7786,2916,4201,3297,3097,2322,2108,2030,4492,1404, 136,1468,1479, 672, // 6486 -1171,3221,2303, 271,3151,7787,2762,7788,2049, 678,2727, 865,1947,4493,7789,2013, // 6502 -3995,2956,7790,2728,2223,1397,3048,3694,4494,4495,1735,2917,3366,3576,7791,3816, // 6518 - 509,2841,2453,2876,3817,7792,7793,3152,3153,4496,4202,2531,4497,2304,1166,1010, // 6534 - 552, 681,1887,7794,7795,2957,2958,3996,1287,1596,1861,3154, 358, 453, 736, 175, // 6550 - 478,1117, 905,1167,1097,7796,1853,1530,7797,1706,7798,2178,3472,2287,3695,3473, // 6566 -3577,4203,2092,4204,7799,3367,1193,2482,4205,1458,2190,2205,1862,1888,1421,3298, // 6582 -2918,3049,2179,3474, 595,2122,7800,3997,7801,7802,4206,1707,2636, 223,3696,1359, // 6598 - 751,3098, 183,3475,7803,2797,3003, 419,2369, 633, 704,3818,2389, 241,7804,7805, // 6614 -7806, 838,3004,3697,2272,2763,2454,3819,1938,2050,3998,1309,3099,2242,1181,7807, // 6630 -1136,2206,3820,2370,1446,4207,2305,4498,7808,7809,4208,1055,2605, 484,3698,7810, // 6646 -3999, 625,4209,2273,3368,1499,4210,4000,7811,4001,4211,3222,2274,2275,3476,7812, // 6662 -7813,2764, 808,2606,3699,3369,4002,4212,3100,2532, 526,3370,3821,4213, 955,7814, // 6678 -1620,4214,2637,2427,7815,1429,3700,1669,1831, 994, 928,7816,3578,1260,7817,7818, // 6694 -7819,1948,2288, 741,2919,1626,4215,2729,2455, 867,1184, 362,3371,1392,7820,7821, // 6710 -4003,4216,1770,1736,3223,2920,4499,4500,1928,2698,1459,1158,7822,3050,3372,2877, // 6726 -1292,1929,2506,2842,3701,1985,1187,2071,2014,2607,4217,7823,2566,2507,2169,3702, // 6742 -2483,3299,7824,3703,4501,7825,7826, 666,1003,3005,1022,3579,4218,7827,4502,1813, // 6758 -2253, 574,3822,1603, 295,1535, 705,3823,4219, 283, 858, 417,7828,7829,3224,4503, // 6774 -4504,3051,1220,1889,1046,2276,2456,4004,1393,1599, 689,2567, 388,4220,7830,2484, // 6790 - 802,7831,2798,3824,2060,1405,2254,7832,4505,3825,2109,1052,1345,3225,1585,7833, // 6806 - 809,7834,7835,7836, 575,2730,3477, 956,1552,1469,1144,2323,7837,2324,1560,2457, // 6822 -3580,3226,4005, 616,2207,3155,2180,2289,7838,1832,7839,3478,4506,7840,1319,3704, // 6838 -3705,1211,3581,1023,3227,1293,2799,7841,7842,7843,3826, 607,2306,3827, 762,2878, // 6854 -1439,4221,1360,7844,1485,3052,7845,4507,1038,4222,1450,2061,2638,4223,1379,4508, // 6870 -2585,7846,7847,4224,1352,1414,2325,2921,1172,7848,7849,3828,3829,7850,1797,1451, // 6886 -7851,7852,7853,7854,2922,4006,4007,2485,2346, 411,4008,4009,3582,3300,3101,4509, // 6902 -1561,2664,1452,4010,1375,7855,7856, 47,2959, 316,7857,1406,1591,2923,3156,7858, // 6918 -1025,2141,3102,3157, 354,2731, 884,2224,4225,2407, 508,3706, 726,3583, 996,2428, // 6934 -3584, 729,7859, 392,2191,1453,4011,4510,3707,7860,7861,2458,3585,2608,1675,2800, // 6950 - 919,2347,2960,2348,1270,4511,4012, 73,7862,7863, 647,7864,3228,2843,2255,1550, // 6966 -1346,3006,7865,1332, 883,3479,7866,7867,7868,7869,3301,2765,7870,1212, 831,1347, // 6982 -4226,4512,2326,3830,1863,3053, 720,3831,4513,4514,3832,7871,4227,7872,7873,4515, // 6998 -7874,7875,1798,4516,3708,2609,4517,3586,1645,2371,7876,7877,2924, 669,2208,2665, // 7014 -2429,7878,2879,7879,7880,1028,3229,7881,4228,2408,7882,2256,1353,7883,7884,4518, // 7030 -3158, 518,7885,4013,7886,4229,1960,7887,2142,4230,7888,7889,3007,2349,2350,3833, // 7046 - 516,1833,1454,4014,2699,4231,4519,2225,2610,1971,1129,3587,7890,2766,7891,2961, // 7062 -1422, 577,1470,3008,1524,3373,7892,7893, 432,4232,3054,3480,7894,2586,1455,2508, // 7078 -2226,1972,1175,7895,1020,2732,4015,3481,4520,7896,2733,7897,1743,1361,3055,3482, // 7094 -2639,4016,4233,4521,2290, 895, 924,4234,2170, 331,2243,3056, 166,1627,3057,1098, // 7110 -7898,1232,2880,2227,3374,4522, 657, 403,1196,2372, 542,3709,3375,1600,4235,3483, // 7126 -7899,4523,2767,3230, 576, 530,1362,7900,4524,2533,2666,3710,4017,7901, 842,3834, // 7142 -7902,2801,2031,1014,4018, 213,2700,3376, 665, 621,4236,7903,3711,2925,2430,7904, // 7158 -2431,3302,3588,3377,7905,4237,2534,4238,4525,3589,1682,4239,3484,1380,7906, 724, // 7174 -2277, 600,1670,7907,1337,1233,4526,3103,2244,7908,1621,4527,7909, 651,4240,7910, // 7190 -1612,4241,2611,7911,2844,7912,2734,2307,3058,7913, 716,2459,3059, 174,1255,2701, // 7206 -4019,3590, 548,1320,1398, 728,4020,1574,7914,1890,1197,3060,4021,7915,3061,3062, // 7222 -3712,3591,3713, 747,7916, 635,4242,4528,7917,7918,7919,4243,7920,7921,4529,7922, // 7238 -3378,4530,2432, 451,7923,3714,2535,2072,4244,2735,4245,4022,7924,1764,4531,7925, // 7254 -4246, 350,7926,2278,2390,2486,7927,4247,4023,2245,1434,4024, 488,4532, 458,4248, // 7270 -4025,3715, 771,1330,2391,3835,2568,3159,2159,2409,1553,2667,3160,4249,7928,2487, // 7286 -2881,2612,1720,2702,4250,3379,4533,7929,2536,4251,7930,3231,4252,2768,7931,2015, // 7302 -2736,7932,1155,1017,3716,3836,7933,3303,2308, 201,1864,4253,1430,7934,4026,7935, // 7318 -7936,7937,7938,7939,4254,1604,7940, 414,1865, 371,2587,4534,4535,3485,2016,3104, // 7334 -4536,1708, 960,4255, 887, 389,2171,1536,1663,1721,7941,2228,4027,2351,2926,1580, // 7350 -7942,7943,7944,1744,7945,2537,4537,4538,7946,4539,7947,2073,7948,7949,3592,3380, // 7366 -2882,4256,7950,4257,2640,3381,2802, 673,2703,2460, 709,3486,4028,3593,4258,7951, // 7382 -1148, 502, 634,7952,7953,1204,4540,3594,1575,4541,2613,3717,7954,3718,3105, 948, // 7398 -3232, 121,1745,3837,1110,7955,4259,3063,2509,3009,4029,3719,1151,1771,3838,1488, // 7414 -4030,1986,7956,2433,3487,7957,7958,2093,7959,4260,3839,1213,1407,2803, 531,2737, // 7430 -2538,3233,1011,1537,7960,2769,4261,3106,1061,7961,3720,3721,1866,2883,7962,2017, // 7446 - 120,4262,4263,2062,3595,3234,2309,3840,2668,3382,1954,4542,7963,7964,3488,1047, // 7462 -2704,1266,7965,1368,4543,2845, 649,3383,3841,2539,2738,1102,2846,2669,7966,7967, // 7478 -1999,7968,1111,3596,2962,7969,2488,3842,3597,2804,1854,3384,3722,7970,7971,3385, // 7494 -2410,2884,3304,3235,3598,7972,2569,7973,3599,2805,4031,1460, 856,7974,3600,7975, // 7510 -2885,2963,7976,2886,3843,7977,4264, 632,2510, 875,3844,1697,3845,2291,7978,7979, // 7526 -4544,3010,1239, 580,4545,4265,7980, 914, 936,2074,1190,4032,1039,2123,7981,7982, // 7542 -7983,3386,1473,7984,1354,4266,3846,7985,2172,3064,4033, 915,3305,4267,4268,3306, // 7558 -1605,1834,7986,2739, 398,3601,4269,3847,4034, 328,1912,2847,4035,3848,1331,4270, // 7574 -3011, 937,4271,7987,3602,4036,4037,3387,2160,4546,3388, 524, 742, 538,3065,1012, // 7590 -7988,7989,3849,2461,7990, 658,1103, 225,3850,7991,7992,4547,7993,4548,7994,3236, // 7606 -1243,7995,4038, 963,2246,4549,7996,2705,3603,3161,7997,7998,2588,2327,7999,4550, // 7622 -8000,8001,8002,3489,3307, 957,3389,2540,2032,1930,2927,2462, 870,2018,3604,1746, // 7638 -2770,2771,2434,2463,8003,3851,8004,3723,3107,3724,3490,3390,3725,8005,1179,3066, // 7654 -8006,3162,2373,4272,3726,2541,3163,3108,2740,4039,8007,3391,1556,2542,2292, 977, // 7670 -2887,2033,4040,1205,3392,8008,1765,3393,3164,2124,1271,1689, 714,4551,3491,8009, // 7686 -2328,3852, 533,4273,3605,2181, 617,8010,2464,3308,3492,2310,8011,8012,3165,8013, // 7702 -8014,3853,1987, 618, 427,2641,3493,3394,8015,8016,1244,1690,8017,2806,4274,4552, // 7718 -8018,3494,8019,8020,2279,1576, 473,3606,4275,3395, 972,8021,3607,8022,3067,8023, // 7734 -8024,4553,4554,8025,3727,4041,4042,8026, 153,4555, 356,8027,1891,2888,4276,2143, // 7750 - 408, 803,2352,8028,3854,8029,4277,1646,2570,2511,4556,4557,3855,8030,3856,4278, // 7766 -8031,2411,3396, 752,8032,8033,1961,2964,8034, 746,3012,2465,8035,4279,3728, 698, // 7782 -4558,1892,4280,3608,2543,4559,3609,3857,8036,3166,3397,8037,1823,1302,4043,2706, // 7798 -3858,1973,4281,8038,4282,3167, 823,1303,1288,1236,2848,3495,4044,3398, 774,3859, // 7814 -8039,1581,4560,1304,2849,3860,4561,8040,2435,2161,1083,3237,4283,4045,4284, 344, // 7830 -1173, 288,2311, 454,1683,8041,8042,1461,4562,4046,2589,8043,8044,4563, 985, 894, // 7846 -8045,3399,3168,8046,1913,2928,3729,1988,8047,2110,1974,8048,4047,8049,2571,1194, // 7862 - 425,8050,4564,3169,1245,3730,4285,8051,8052,2850,8053, 636,4565,1855,3861, 760, // 7878 -1799,8054,4286,2209,1508,4566,4048,1893,1684,2293,8055,8056,8057,4287,4288,2210, // 7894 - 479,8058,8059, 832,8060,4049,2489,8061,2965,2490,3731, 990,3109, 627,1814,2642, // 7910 -4289,1582,4290,2125,2111,3496,4567,8062, 799,4291,3170,8063,4568,2112,1737,3013, // 7926 -1018, 543, 754,4292,3309,1676,4569,4570,4050,8064,1489,8065,3497,8066,2614,2889, // 7942 -4051,8067,8068,2966,8069,8070,8071,8072,3171,4571,4572,2182,1722,8073,3238,3239, // 7958 -1842,3610,1715, 481, 365,1975,1856,8074,8075,1962,2491,4573,8076,2126,3611,3240, // 7974 - 433,1894,2063,2075,8077, 602,2741,8078,8079,8080,8081,8082,3014,1628,3400,8083, // 7990 -3172,4574,4052,2890,4575,2512,8084,2544,2772,8085,8086,8087,3310,4576,2891,8088, // 8006 -4577,8089,2851,4578,4579,1221,2967,4053,2513,8090,8091,8092,1867,1989,8093,8094, // 8022 -8095,1895,8096,8097,4580,1896,4054, 318,8098,2094,4055,4293,8099,8100, 485,8101, // 8038 - 938,3862, 553,2670, 116,8102,3863,3612,8103,3498,2671,2773,3401,3311,2807,8104, // 8054 -3613,2929,4056,1747,2930,2968,8105,8106, 207,8107,8108,2672,4581,2514,8109,3015, // 8070 - 890,3614,3864,8110,1877,3732,3402,8111,2183,2353,3403,1652,8112,8113,8114, 941, // 8086 -2294, 208,3499,4057,2019, 330,4294,3865,2892,2492,3733,4295,8115,8116,8117,8118, // 8102 - -/*************************************************************************************** - *Everything below is of no interest for detection purpose * - *************************************************************************************** - -2515,1613,4582,8119,3312,3866,2516,8120,4058,8121,1637,4059,2466,4583,3867,8122, // 8118 -2493,3016,3734,8123,8124,2192,8125,8126,2162,8127,8128,8129,8130,8131,8132,8133, // 8134 -8134,8135,8136,8137,8138,8139,8140,8141,8142,8143,8144,8145,8146,8147,8148,8149, // 8150 -8150,8151,8152,8153,8154,8155,8156,8157,8158,8159,8160,8161,8162,8163,8164,8165, // 8166 -8166,8167,8168,8169,8170,8171,8172,8173,8174,8175,8176,8177,8178,8179,8180,8181, // 8182 -8182,8183,8184,8185,8186,8187,8188,8189,8190,8191,8192,8193,8194,8195,8196,8197, // 8198 -8198,8199,8200,8201,8202,8203,8204,8205,8206,8207,8208,8209,8210,8211,8212,8213, // 8214 -8214,8215,8216,8217,8218,8219,8220,8221,8222,8223,8224,8225,8226,8227,8228,8229, // 8230 -8230,8231,8232,8233,8234,8235,8236,8237,8238,8239,8240,8241,8242,8243,8244,8245, // 8246 -8246,8247,8248,8249,8250,8251,8252,8253,8254,8255,8256,8257,8258,8259,8260,8261, // 8262 -8262,8263,8264,8265,8266,8267,8268,8269,8270,8271,8272,8273,8274,8275,8276,8277, // 8278 -8278,8279,8280,8281,8282,8283,8284,8285,8286,8287,8288,8289,8290,8291,8292,8293, // 8294 -8294,8295,8296,8297,8298,8299,8300,8301,8302,8303,8304,8305,8306,8307,8308,8309, // 8310 -8310,8311,8312,8313,8314,8315,8316,8317,8318,8319,8320,8321,8322,8323,8324,8325, // 8326 -8326,8327,8328,8329,8330,8331,8332,8333,8334,8335,8336,8337,8338,8339,8340,8341, // 8342 -8342,8343,8344,8345,8346,8347,8348,8349,8350,8351,8352,8353,8354,8355,8356,8357, // 8358 -8358,8359,8360,8361,8362,8363,8364,8365,8366,8367,8368,8369,8370,8371,8372,8373, // 8374 -8374,8375,8376,8377,8378,8379,8380,8381,8382,8383,8384,8385,8386,8387,8388,8389, // 8390 -8390,8391,8392,8393,8394,8395,8396,8397,8398,8399,8400,8401,8402,8403,8404,8405, // 8406 -8406,8407,8408,8409,8410,8411,8412,8413,8414,8415,8416,8417,8418,8419,8420,8421, // 8422 -8422,8423,8424,8425,8426,8427,8428,8429,8430,8431,8432,8433,8434,8435,8436,8437, // 8438 -8438,8439,8440,8441,8442,8443,8444,8445,8446,8447,8448,8449,8450,8451,8452,8453, // 8454 -8454,8455,8456,8457,8458,8459,8460,8461,8462,8463,8464,8465,8466,8467,8468,8469, // 8470 -8470,8471,8472,8473,8474,8475,8476,8477,8478,8479,8480,8481,8482,8483,8484,8485, // 8486 -8486,8487,8488,8489,8490,8491,8492,8493,8494,8495,8496,8497,8498,8499,8500,8501, // 8502 -8502,8503,8504,8505,8506,8507,8508,8509,8510,8511,8512,8513,8514,8515,8516,8517, // 8518 -8518,8519,8520,8521,8522,8523,8524,8525,8526,8527,8528,8529,8530,8531,8532,8533, // 8534 -8534,8535,8536,8537,8538,8539,8540,8541,8542,8543,8544,8545,8546,8547,8548,8549, // 8550 -8550,8551,8552,8553,8554,8555,8556,8557,8558,8559,8560,8561,8562,8563,8564,8565, // 8566 -8566,8567,8568,8569,8570,8571,8572,8573,8574,8575,8576,8577,8578,8579,8580,8581, // 8582 -8582,8583,8584,8585,8586,8587,8588,8589,8590,8591,8592,8593,8594,8595,8596,8597, // 8598 -8598,8599,8600,8601,8602,8603,8604,8605,8606,8607,8608,8609,8610,8611,8612,8613, // 8614 -8614,8615,8616,8617,8618,8619,8620,8621,8622,8623,8624,8625,8626,8627,8628,8629, // 8630 -8630,8631,8632,8633,8634,8635,8636,8637,8638,8639,8640,8641,8642,8643,8644,8645, // 8646 -8646,8647,8648,8649,8650,8651,8652,8653,8654,8655,8656,8657,8658,8659,8660,8661, // 8662 -8662,8663,8664,8665,8666,8667,8668,8669,8670,8671,8672,8673,8674,8675,8676,8677, // 8678 -8678,8679,8680,8681,8682,8683,8684,8685,8686,8687,8688,8689,8690,8691,8692,8693, // 8694 -8694,8695,8696,8697,8698,8699,8700,8701,8702,8703,8704,8705,8706,8707,8708,8709, // 8710 -8710,8711,8712,8713,8714,8715,8716,8717,8718,8719,8720,8721,8722,8723,8724,8725, // 8726 -8726,8727,8728,8729,8730,8731,8732,8733,8734,8735,8736,8737,8738,8739,8740,8741, // 8742 //13973 -****************************************************************************************/ -}; - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/GB2312Freq.tab b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/GB2312Freq.tab deleted file mode 100644 index aee12ca..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/GB2312Freq.tab +++ /dev/null @@ -1,491 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -//GB2312 most frequently used character table - -//Char to FreqOrder table , from hz6763 - -/****************************************************************************** - * 512 --> 0.79 -- 0.79 - * 1024 --> 0.92 -- 0.13 - * 2048 --> 0.98 -- 0.06 - * 6768 --> 1.00 -- 0.02 - * - * Idea Distribution Ratio = 0.79135/(1-0.79135) = 3.79 - * Random Distribution Ration = 512 / (3755 - 512) = 0.157 - * - * Typical Distribution Ratio about 25% of Ideal one, still much higher that RDR - *****************************************************************************/ - -#define GB2312_TYPICAL_DISTRIBUTION_RATIO (float)0.9 - -#define GB2312_TABLE_SIZE 3760 - -static const PRInt16 GB2312CharToFreqOrder[] = -{ -1671, 749,1443,2364,3924,3807,2330,3921,1704,3463,2691,1511,1515, 572,3191,2205, -2361, 224,2558, 479,1711, 963,3162, 440,4060,1905,2966,2947,3580,2647,3961,3842, -2204, 869,4207, 970,2678,5626,2944,2956,1479,4048, 514,3595, 588,1346,2820,3409, - 249,4088,1746,1873,2047,1774, 581,1813, 358,1174,3590,1014,1561,4844,2245, 670, -1636,3112, 889,1286, 953, 556,2327,3060,1290,3141, 613, 185,3477,1367, 850,3820, -1715,2428,2642,2303,2732,3041,2562,2648,3566,3946,1349, 388,3098,2091,1360,3585, - 152,1687,1539, 738,1559, 59,1232,2925,2267,1388,1249,1741,1679,2960, 151,1566, -1125,1352,4271, 924,4296, 385,3166,4459, 310,1245,2850, 70,3285,2729,3534,3575, -2398,3298,3466,1960,2265, 217,3647, 864,1909,2084,4401,2773,1010,3269,5152, 853, -3051,3121,1244,4251,1895, 364,1499,1540,2313,1180,3655,2268, 562, 715,2417,3061, - 544, 336,3768,2380,1752,4075, 950, 280,2425,4382, 183,2759,3272, 333,4297,2155, -1688,2356,1444,1039,4540, 736,1177,3349,2443,2368,2144,2225, 565, 196,1482,3406, - 927,1335,4147, 692, 878,1311,1653,3911,3622,1378,4200,1840,2969,3149,2126,1816, -2534,1546,2393,2760, 737,2494, 13, 447, 245,2747, 38,2765,2129,2589,1079, 606, - 360, 471,3755,2890, 404, 848, 699,1785,1236, 370,2221,1023,3746,2074,2026,2023, -2388,1581,2119, 812,1141,3091,2536,1519, 804,2053, 406,1596,1090, 784, 548,4414, -1806,2264,2936,1100, 343,4114,5096, 622,3358, 743,3668,1510,1626,5020,3567,2513, -3195,4115,5627,2489,2991, 24,2065,2697,1087,2719, 48,1634, 315, 68, 985,2052, - 198,2239,1347,1107,1439, 597,2366,2172, 871,3307, 919,2487,2790,1867, 236,2570, -1413,3794, 906,3365,3381,1701,1982,1818,1524,2924,1205, 616,2586,2072,2004, 575, - 253,3099, 32,1365,1182, 197,1714,2454,1201, 554,3388,3224,2748, 756,2587, 250, -2567,1507,1517,3529,1922,2761,2337,3416,1961,1677,2452,2238,3153, 615, 911,1506, -1474,2495,1265,1906,2749,3756,3280,2161, 898,2714,1759,3450,2243,2444, 563, 26, -3286,2266,3769,3344,2707,3677, 611,1402, 531,1028,2871,4548,1375, 261,2948, 835, -1190,4134, 353, 840,2684,1900,3082,1435,2109,1207,1674, 329,1872,2781,4055,2686, -2104, 608,3318,2423,2957,2768,1108,3739,3512,3271,3985,2203,1771,3520,1418,2054, -1681,1153, 225,1627,2929, 162,2050,2511,3687,1954, 124,1859,2431,1684,3032,2894, - 585,4805,3969,2869,2704,2088,2032,2095,3656,2635,4362,2209, 256, 518,2042,2105, -3777,3657, 643,2298,1148,1779, 190, 989,3544, 414, 11,2135,2063,2979,1471, 403, -3678, 126, 770,1563, 671,2499,3216,2877, 600,1179, 307,2805,4937,1268,1297,2694, - 252,4032,1448,1494,1331,1394, 127,2256, 222,1647,1035,1481,3056,1915,1048, 873, -3651, 210, 33,1608,2516, 200,1520, 415, 102, 0,3389,1287, 817, 91,3299,2940, - 836,1814, 549,2197,1396,1669,2987,3582,2297,2848,4528,1070, 687, 20,1819, 121, -1552,1364,1461,1968,2617,3540,2824,2083, 177, 948,4938,2291, 110,4549,2066, 648, -3359,1755,2110,2114,4642,4845,1693,3937,3308,1257,1869,2123, 208,1804,3159,2992, -2531,2549,3361,2418,1350,2347,2800,2568,1291,2036,2680, 72, 842,1990, 212,1233, -1154,1586, 75,2027,3410,4900,1823,1337,2710,2676, 728,2810,1522,3026,4995, 157, - 755,1050,4022, 710, 785,1936,2194,2085,1406,2777,2400, 150,1250,4049,1206, 807, -1910, 534, 529,3309,1721,1660, 274, 39,2827, 661,2670,1578, 925,3248,3815,1094, -4278,4901,4252, 41,1150,3747,2572,2227,4501,3658,4902,3813,3357,3617,2884,2258, - 887, 538,4187,3199,1294,2439,3042,2329,2343,2497,1255, 107, 543,1527, 521,3478, -3568, 194,5062, 15, 961,3870,1241,1192,2664, 66,5215,3260,2111,1295,1127,2152, -3805,4135, 901,1164,1976, 398,1278, 530,1460, 748, 904,1054,1966,1426, 53,2909, - 509, 523,2279,1534, 536,1019, 239,1685, 460,2353, 673,1065,2401,3600,4298,2272, -1272,2363, 284,1753,3679,4064,1695, 81, 815,2677,2757,2731,1386, 859, 500,4221, -2190,2566, 757,1006,2519,2068,1166,1455, 337,2654,3203,1863,1682,1914,3025,1252, -1409,1366, 847, 714,2834,2038,3209, 964,2970,1901, 885,2553,1078,1756,3049, 301, -1572,3326, 688,2130,1996,2429,1805,1648,2930,3421,2750,3652,3088, 262,1158,1254, - 389,1641,1812, 526,1719, 923,2073,1073,1902, 468, 489,4625,1140, 857,2375,3070, -3319,2863, 380, 116,1328,2693,1161,2244, 273,1212,1884,2769,3011,1775,1142, 461, -3066,1200,2147,2212, 790, 702,2695,4222,1601,1058, 434,2338,5153,3640, 67,2360, -4099,2502, 618,3472,1329, 416,1132, 830,2782,1807,2653,3211,3510,1662, 192,2124, - 296,3979,1739,1611,3684, 23, 118, 324, 446,1239,1225, 293,2520,3814,3795,2535, -3116, 17,1074, 467,2692,2201, 387,2922, 45,1326,3055,1645,3659,2817, 958, 243, -1903,2320,1339,2825,1784,3289, 356, 576, 865,2315,2381,3377,3916,1088,3122,1713, -1655, 935, 628,4689,1034,1327, 441, 800, 720, 894,1979,2183,1528,5289,2702,1071, -4046,3572,2399,1571,3281, 79, 761,1103, 327, 134, 758,1899,1371,1615, 879, 442, - 215,2605,2579, 173,2048,2485,1057,2975,3317,1097,2253,3801,4263,1403,1650,2946, - 814,4968,3487,1548,2644,1567,1285, 2, 295,2636, 97, 946,3576, 832, 141,4257, -3273, 760,3821,3521,3156,2607, 949,1024,1733,1516,1803,1920,2125,2283,2665,3180, -1501,2064,3560,2171,1592, 803,3518,1416, 732,3897,4258,1363,1362,2458, 119,1427, - 602,1525,2608,1605,1639,3175, 694,3064, 10, 465, 76,2000,4846,4208, 444,3781, -1619,3353,2206,1273,3796, 740,2483, 320,1723,2377,3660,2619,1359,1137,1762,1724, -2345,2842,1850,1862, 912, 821,1866, 612,2625,1735,2573,3369,1093, 844, 89, 937, - 930,1424,3564,2413,2972,1004,3046,3019,2011, 711,3171,1452,4178, 428, 801,1943, - 432, 445,2811, 206,4136,1472, 730, 349, 73, 397,2802,2547, 998,1637,1167, 789, - 396,3217, 154,1218, 716,1120,1780,2819,4826,1931,3334,3762,2139,1215,2627, 552, -3664,3628,3232,1405,2383,3111,1356,2652,3577,3320,3101,1703, 640,1045,1370,1246, -4996, 371,1575,2436,1621,2210, 984,4033,1734,2638, 16,4529, 663,2755,3255,1451, -3917,2257,1253,1955,2234,1263,2951, 214,1229, 617, 485, 359,1831,1969, 473,2310, - 750,2058, 165, 80,2864,2419, 361,4344,2416,2479,1134, 796,3726,1266,2943, 860, -2715, 938, 390,2734,1313,1384, 248, 202, 877,1064,2854, 522,3907, 279,1602, 297, -2357, 395,3740, 137,2075, 944,4089,2584,1267,3802, 62,1533,2285, 178, 176, 780, -2440, 201,3707, 590, 478,1560,4354,2117,1075, 30, 74,4643,4004,1635,1441,2745, - 776,2596, 238,1077,1692,1912,2844, 605, 499,1742,3947, 241,3053, 980,1749, 936, -2640,4511,2582, 515,1543,2162,5322,2892,2993, 890,2148,1924, 665,1827,3581,1032, - 968,3163, 339,1044,1896, 270, 583,1791,1720,4367,1194,3488,3669, 43,2523,1657, - 163,2167, 290,1209,1622,3378, 550, 634,2508,2510, 695,2634,2384,2512,1476,1414, - 220,1469,2341,2138,2852,3183,2900,4939,2865,3502,1211,3680, 854,3227,1299,2976, -3172, 186,2998,1459, 443,1067,3251,1495, 321,1932,3054, 909, 753,1410,1828, 436, -2441,1119,1587,3164,2186,1258, 227, 231,1425,1890,3200,3942, 247, 959, 725,5254, -2741, 577,2158,2079, 929, 120, 174, 838,2813, 591,1115, 417,2024, 40,3240,1536, -1037, 291,4151,2354, 632,1298,2406,2500,3535,1825,1846,3451, 205,1171, 345,4238, - 18,1163, 811, 685,2208,1217, 425,1312,1508,1175,4308,2552,1033, 587,1381,3059, -2984,3482, 340,1316,4023,3972, 792,3176, 519, 777,4690, 918, 933,4130,2981,3741, - 90,3360,2911,2200,5184,4550, 609,3079,2030, 272,3379,2736, 363,3881,1130,1447, - 286, 779, 357,1169,3350,3137,1630,1220,2687,2391, 747,1277,3688,2618,2682,2601, -1156,3196,5290,4034,3102,1689,3596,3128, 874, 219,2783, 798, 508,1843,2461, 269, -1658,1776,1392,1913,2983,3287,2866,2159,2372, 829,4076, 46,4253,2873,1889,1894, - 915,1834,1631,2181,2318, 298, 664,2818,3555,2735, 954,3228,3117, 527,3511,2173, - 681,2712,3033,2247,2346,3467,1652, 155,2164,3382, 113,1994, 450, 899, 494, 994, -1237,2958,1875,2336,1926,3727, 545,1577,1550, 633,3473, 204,1305,3072,2410,1956, -2471, 707,2134, 841,2195,2196,2663,3843,1026,4940, 990,3252,4997, 368,1092, 437, -3212,3258,1933,1829, 675,2977,2893, 412, 943,3723,4644,3294,3283,2230,2373,5154, -2389,2241,2661,2323,1404,2524, 593, 787, 677,3008,1275,2059, 438,2709,2609,2240, -2269,2246,1446, 36,1568,1373,3892,1574,2301,1456,3962, 693,2276,5216,2035,1143, -2720,1919,1797,1811,2763,4137,2597,1830,1699,1488,1198,2090, 424,1694, 312,3634, -3390,4179,3335,2252,1214, 561,1059,3243,2295,2561, 975,5155,2321,2751,3772, 472, -1537,3282,3398,1047,2077,2348,2878,1323,3340,3076, 690,2906, 51, 369, 170,3541, -1060,2187,2688,3670,2541,1083,1683, 928,3918, 459, 109,4427, 599,3744,4286, 143, -2101,2730,2490, 82,1588,3036,2121, 281,1860, 477,4035,1238,2812,3020,2716,3312, -1530,2188,2055,1317, 843, 636,1808,1173,3495, 649, 181,1002, 147,3641,1159,2414, -3750,2289,2795, 813,3123,2610,1136,4368, 5,3391,4541,2174, 420, 429,1728, 754, -1228,2115,2219, 347,2223,2733, 735,1518,3003,2355,3134,1764,3948,3329,1888,2424, -1001,1234,1972,3321,3363,1672,1021,1450,1584, 226, 765, 655,2526,3404,3244,2302, -3665, 731, 594,2184, 319,1576, 621, 658,2656,4299,2099,3864,1279,2071,2598,2739, - 795,3086,3699,3908,1707,2352,2402,1382,3136,2475,1465,4847,3496,3865,1085,3004, -2591,1084, 213,2287,1963,3565,2250, 822, 793,4574,3187,1772,1789,3050, 595,1484, -1959,2770,1080,2650, 456, 422,2996, 940,3322,4328,4345,3092,2742, 965,2784, 739, -4124, 952,1358,2498,2949,2565, 332,2698,2378, 660,2260,2473,4194,3856,2919, 535, -1260,2651,1208,1428,1300,1949,1303,2942, 433,2455,2450,1251,1946, 614,1269, 641, -1306,1810,2737,3078,2912, 564,2365,1419,1415,1497,4460,2367,2185,1379,3005,1307, -3218,2175,1897,3063, 682,1157,4040,4005,1712,1160,1941,1399, 394, 402,2952,1573, -1151,2986,2404, 862, 299,2033,1489,3006, 346, 171,2886,3401,1726,2932, 168,2533, - 47,2507,1030,3735,1145,3370,1395,1318,1579,3609,4560,2857,4116,1457,2529,1965, - 504,1036,2690,2988,2405, 745,5871, 849,2397,2056,3081, 863,2359,3857,2096, 99, -1397,1769,2300,4428,1643,3455,1978,1757,3718,1440, 35,4879,3742,1296,4228,2280, - 160,5063,1599,2013, 166, 520,3479,1646,3345,3012, 490,1937,1545,1264,2182,2505, -1096,1188,1369,1436,2421,1667,2792,2460,1270,2122, 727,3167,2143, 806,1706,1012, -1800,3037, 960,2218,1882, 805, 139,2456,1139,1521, 851,1052,3093,3089, 342,2039, - 744,5097,1468,1502,1585,2087, 223, 939, 326,2140,2577, 892,2481,1623,4077, 982, -3708, 135,2131, 87,2503,3114,2326,1106, 876,1616, 547,2997,2831,2093,3441,4530, -4314, 9,3256,4229,4148, 659,1462,1986,1710,2046,2913,2231,4090,4880,5255,3392, -3274,1368,3689,4645,1477, 705,3384,3635,1068,1529,2941,1458,3782,1509, 100,1656, -2548, 718,2339, 408,1590,2780,3548,1838,4117,3719,1345,3530, 717,3442,2778,3220, -2898,1892,4590,3614,3371,2043,1998,1224,3483, 891, 635, 584,2559,3355, 733,1766, -1729,1172,3789,1891,2307, 781,2982,2271,1957,1580,5773,2633,2005,4195,3097,1535, -3213,1189,1934,5693,3262, 586,3118,1324,1598, 517,1564,2217,1868,1893,4445,3728, -2703,3139,1526,1787,1992,3882,2875,1549,1199,1056,2224,1904,2711,5098,4287, 338, -1993,3129,3489,2689,1809,2815,1997, 957,1855,3898,2550,3275,3057,1105,1319, 627, -1505,1911,1883,3526, 698,3629,3456,1833,1431, 746, 77,1261,2017,2296,1977,1885, - 125,1334,1600, 525,1798,1109,2222,1470,1945, 559,2236,1186,3443,2476,1929,1411, -2411,3135,1777,3372,2621,1841,1613,3229, 668,1430,1839,2643,2916, 195,1989,2671, -2358,1387, 629,3205,2293,5256,4439, 123,1310, 888,1879,4300,3021,3605,1003,1162, -3192,2910,2010, 140,2395,2859, 55,1082,2012,2901, 662, 419,2081,1438, 680,2774, -4654,3912,1620,1731,1625,5035,4065,2328, 512,1344, 802,5443,2163,2311,2537, 524, -3399, 98,1155,2103,1918,2606,3925,2816,1393,2465,1504,3773,2177,3963,1478,4346, - 180,1113,4655,3461,2028,1698, 833,2696,1235,1322,1594,4408,3623,3013,3225,2040, -3022, 541,2881, 607,3632,2029,1665,1219, 639,1385,1686,1099,2803,3231,1938,3188, -2858, 427, 676,2772,1168,2025, 454,3253,2486,3556, 230,1950, 580, 791,1991,1280, -1086,1974,2034, 630, 257,3338,2788,4903,1017, 86,4790, 966,2789,1995,1696,1131, - 259,3095,4188,1308, 179,1463,5257, 289,4107,1248, 42,3413,1725,2288, 896,1947, - 774,4474,4254, 604,3430,4264, 392,2514,2588, 452, 237,1408,3018, 988,4531,1970, -3034,3310, 540,2370,1562,1288,2990, 502,4765,1147, 4,1853,2708, 207, 294,2814, -4078,2902,2509, 684, 34,3105,3532,2551, 644, 709,2801,2344, 573,1727,3573,3557, -2021,1081,3100,4315,2100,3681, 199,2263,1837,2385, 146,3484,1195,2776,3949, 997, -1939,3973,1008,1091,1202,1962,1847,1149,4209,5444,1076, 493, 117,5400,2521, 972, -1490,2934,1796,4542,2374,1512,2933,2657, 413,2888,1135,2762,2314,2156,1355,2369, - 766,2007,2527,2170,3124,2491,2593,2632,4757,2437, 234,3125,3591,1898,1750,1376, -1942,3468,3138, 570,2127,2145,3276,4131, 962, 132,1445,4196, 19, 941,3624,3480, -3366,1973,1374,4461,3431,2629, 283,2415,2275, 808,2887,3620,2112,2563,1353,3610, - 955,1089,3103,1053, 96, 88,4097, 823,3808,1583, 399, 292,4091,3313, 421,1128, - 642,4006, 903,2539,1877,2082, 596, 29,4066,1790, 722,2157, 130, 995,1569, 769, -1485, 464, 513,2213, 288,1923,1101,2453,4316, 133, 486,2445, 50, 625, 487,2207, - 57, 423, 481,2962, 159,3729,1558, 491, 303, 482, 501, 240,2837, 112,3648,2392, -1783, 362, 8,3433,3422, 610,2793,3277,1390,1284,1654, 21,3823, 734, 367, 623, - 193, 287, 374,1009,1483, 816, 476, 313,2255,2340,1262,2150,2899,1146,2581, 782, -2116,1659,2018,1880, 255,3586,3314,1110,2867,2137,2564, 986,2767,5185,2006, 650, - 158, 926, 762, 881,3157,2717,2362,3587, 306,3690,3245,1542,3077,2427,1691,2478, -2118,2985,3490,2438, 539,2305, 983, 129,1754, 355,4201,2386, 827,2923, 104,1773, -2838,2771, 411,2905,3919, 376, 767, 122,1114, 828,2422,1817,3506, 266,3460,1007, -1609,4998, 945,2612,4429,2274, 726,1247,1964,2914,2199,2070,4002,4108, 657,3323, -1422, 579, 455,2764,4737,1222,2895,1670, 824,1223,1487,2525, 558, 861,3080, 598, -2659,2515,1967, 752,2583,2376,2214,4180, 977, 704,2464,4999,2622,4109,1210,2961, - 819,1541, 142,2284, 44, 418, 457,1126,3730,4347,4626,1644,1876,3671,1864, 302, -1063,5694, 624, 723,1984,3745,1314,1676,2488,1610,1449,3558,3569,2166,2098, 409, -1011,2325,3704,2306, 818,1732,1383,1824,1844,3757, 999,2705,3497,1216,1423,2683, -2426,2954,2501,2726,2229,1475,2554,5064,1971,1794,1666,2014,1343, 783, 724, 191, -2434,1354,2220,5065,1763,2752,2472,4152, 131, 175,2885,3434, 92,1466,4920,2616, -3871,3872,3866, 128,1551,1632, 669,1854,3682,4691,4125,1230, 188,2973,3290,1302, -1213, 560,3266, 917, 763,3909,3249,1760, 868,1958, 764,1782,2097, 145,2277,3774, -4462, 64,1491,3062, 971,2132,3606,2442, 221,1226,1617, 218, 323,1185,3207,3147, - 571, 619,1473,1005,1744,2281, 449,1887,2396,3685, 275, 375,3816,1743,3844,3731, - 845,1983,2350,4210,1377, 773, 967,3499,3052,3743,2725,4007,1697,1022,3943,1464, -3264,2855,2722,1952,1029,2839,2467, 84,4383,2215, 820,1391,2015,2448,3672, 377, -1948,2168, 797,2545,3536,2578,2645, 94,2874,1678, 405,1259,3071, 771, 546,1315, - 470,1243,3083, 895,2468, 981, 969,2037, 846,4181, 653,1276,2928, 14,2594, 557, -3007,2474, 156, 902,1338,1740,2574, 537,2518, 973,2282,2216,2433,1928, 138,2903, -1293,2631,1612, 646,3457, 839,2935, 111, 496,2191,2847, 589,3186, 149,3994,2060, -4031,2641,4067,3145,1870, 37,3597,2136,1025,2051,3009,3383,3549,1121,1016,3261, -1301, 251,2446,2599,2153, 872,3246, 637, 334,3705, 831, 884, 921,3065,3140,4092, -2198,1944, 246,2964, 108,2045,1152,1921,2308,1031, 203,3173,4170,1907,3890, 810, -1401,2003,1690, 506, 647,1242,2828,1761,1649,3208,2249,1589,3709,2931,5156,1708, - 498, 666,2613, 834,3817,1231, 184,2851,1124, 883,3197,2261,3710,1765,1553,2658, -1178,2639,2351, 93,1193, 942,2538,2141,4402, 235,1821, 870,1591,2192,1709,1871, -3341,1618,4126,2595,2334, 603, 651, 69, 701, 268,2662,3411,2555,1380,1606, 503, - 448, 254,2371,2646, 574,1187,2309,1770, 322,2235,1292,1801, 305, 566,1133, 229, -2067,2057, 706, 167, 483,2002,2672,3295,1820,3561,3067, 316, 378,2746,3452,1112, - 136,1981, 507,1651,2917,1117, 285,4591, 182,2580,3522,1304, 335,3303,1835,2504, -1795,1792,2248, 674,1018,2106,2449,1857,2292,2845, 976,3047,1781,2600,2727,1389, -1281, 52,3152, 153, 265,3950, 672,3485,3951,4463, 430,1183, 365, 278,2169, 27, -1407,1336,2304, 209,1340,1730,2202,1852,2403,2883, 979,1737,1062, 631,2829,2542, -3876,2592, 825,2086,2226,3048,3625, 352,1417,3724, 542, 991, 431,1351,3938,1861, -2294, 826,1361,2927,3142,3503,1738, 463,2462,2723, 582,1916,1595,2808, 400,3845, -3891,2868,3621,2254, 58,2492,1123, 910,2160,2614,1372,1603,1196,1072,3385,1700, -3267,1980, 696, 480,2430, 920, 799,1570,2920,1951,2041,4047,2540,1321,4223,2469, -3562,2228,1271,2602, 401,2833,3351,2575,5157, 907,2312,1256, 410, 263,3507,1582, - 996, 678,1849,2316,1480, 908,3545,2237, 703,2322, 667,1826,2849,1531,2604,2999, -2407,3146,2151,2630,1786,3711, 469,3542, 497,3899,2409, 858, 837,4446,3393,1274, - 786, 620,1845,2001,3311, 484, 308,3367,1204,1815,3691,2332,1532,2557,1842,2020, -2724,1927,2333,4440, 567, 22,1673,2728,4475,1987,1858,1144,1597, 101,1832,3601, - 12, 974,3783,4391, 951,1412, 1,3720, 453,4608,4041, 528,1041,1027,3230,2628, -1129, 875,1051,3291,1203,2262,1069,2860,2799,2149,2615,3278, 144,1758,3040, 31, - 475,1680, 366,2685,3184, 311,1642,4008,2466,5036,1593,1493,2809, 216,1420,1668, - 233, 304,2128,3284, 232,1429,1768,1040,2008,3407,2740,2967,2543, 242,2133, 778, -1565,2022,2620, 505,2189,2756,1098,2273, 372,1614, 708, 553,2846,2094,2278, 169, -3626,2835,4161, 228,2674,3165, 809,1454,1309, 466,1705,1095, 900,3423, 880,2667, -3751,5258,2317,3109,2571,4317,2766,1503,1342, 866,4447,1118, 63,2076, 314,1881, -1348,1061, 172, 978,3515,1747, 532, 511,3970, 6, 601, 905,2699,3300,1751, 276, -1467,3725,2668, 65,4239,2544,2779,2556,1604, 578,2451,1802, 992,2331,2624,1320, -3446, 713,1513,1013, 103,2786,2447,1661, 886,1702, 916, 654,3574,2031,1556, 751, -2178,2821,2179,1498,1538,2176, 271, 914,2251,2080,1325, 638,1953,2937,3877,2432, -2754, 95,3265,1716, 260,1227,4083, 775, 106,1357,3254, 426,1607, 555,2480, 772, -1985, 244,2546, 474, 495,1046,2611,1851,2061, 71,2089,1675,2590, 742,3758,2843, -3222,1433, 267,2180,2576,2826,2233,2092,3913,2435, 956,1745,3075, 856,2113,1116, - 451, 3,1988,2896,1398, 993,2463,1878,2049,1341,2718,2721,2870,2108, 712,2904, -4363,2753,2324, 277,2872,2349,2649, 384, 987, 435, 691,3000, 922, 164,3939, 652, -1500,1184,4153,2482,3373,2165,4848,2335,3775,3508,3154,2806,2830,1554,2102,1664, -2530,1434,2408, 893,1547,2623,3447,2832,2242,2532,3169,2856,3223,2078, 49,3770, -3469, 462, 318, 656,2259,3250,3069, 679,1629,2758, 344,1138,1104,3120,1836,1283, -3115,2154,1437,4448, 934, 759,1999, 794,2862,1038, 533,2560,1722,2342, 855,2626, -1197,1663,4476,3127, 85,4240,2528, 25,1111,1181,3673, 407,3470,4561,2679,2713, - 768,1925,2841,3986,1544,1165, 932, 373,1240,2146,1930,2673, 721,4766, 354,4333, - 391,2963, 187, 61,3364,1442,1102, 330,1940,1767, 341,3809,4118, 393,2496,2062, -2211, 105, 331, 300, 439, 913,1332, 626, 379,3304,1557, 328, 689,3952, 309,1555, - 931, 317,2517,3027, 325, 569, 686,2107,3084, 60,1042,1333,2794, 264,3177,4014, -1628, 258,3712, 7,4464,1176,1043,1778, 683, 114,1975, 78,1492, 383,1886, 510, - 386, 645,5291,2891,2069,3305,4138,3867,2939,2603,2493,1935,1066,1848,3588,1015, -1282,1289,4609, 697,1453,3044,2666,3611,1856,2412, 54, 719,1330, 568,3778,2459, -1748, 788, 492, 551,1191,1000, 488,3394,3763, 282,1799, 348,2016,1523,3155,2390, -1049, 382,2019,1788,1170, 729,2968,3523, 897,3926,2785,2938,3292, 350,2319,3238, -1718,1717,2655,3453,3143,4465, 161,2889,2980,2009,1421, 56,1908,1640,2387,2232, -1917,1874,2477,4921, 148, 83,3438, 592,4245,2882,1822,1055, 741, 115,1496,1624, - 381,1638,4592,1020, 516,3214, 458, 947,4575,1432, 211,1514,2926,1865,2142, 189, - 852,1221,1400,1486, 882,2299,4036, 351, 28,1122, 700,6479,6480,6481,6482,6483, //last 512 - -/*************************************************************************************** - *Everything below is of no interest for detection purpose * - *************************************************************************************** - -5508,6484,3900,3414,3974,4441,4024,3537,4037,5628,5099,3633,6485,3148,6486,3636, -5509,3257,5510,5973,5445,5872,4941,4403,3174,4627,5873,6276,2286,4230,5446,5874, -5122,6102,6103,4162,5447,5123,5323,4849,6277,3980,3851,5066,4246,5774,5067,6278, -3001,2807,5695,3346,5775,5974,5158,5448,6487,5975,5976,5776,3598,6279,5696,4806, -4211,4154,6280,6488,6489,6490,6281,4212,5037,3374,4171,6491,4562,4807,4722,4827, -5977,6104,4532,4079,5159,5324,5160,4404,3858,5359,5875,3975,4288,4610,3486,4512, -5325,3893,5360,6282,6283,5560,2522,4231,5978,5186,5449,2569,3878,6284,5401,3578, -4415,6285,4656,5124,5979,2506,4247,4449,3219,3417,4334,4969,4329,6492,4576,4828, -4172,4416,4829,5402,6286,3927,3852,5361,4369,4830,4477,4867,5876,4173,6493,6105, -4657,6287,6106,5877,5450,6494,4155,4868,5451,3700,5629,4384,6288,6289,5878,3189, -4881,6107,6290,6495,4513,6496,4692,4515,4723,5100,3356,6497,6291,3810,4080,5561, -3570,4430,5980,6498,4355,5697,6499,4724,6108,6109,3764,4050,5038,5879,4093,3226, -6292,5068,5217,4693,3342,5630,3504,4831,4377,4466,4309,5698,4431,5777,6293,5778, -4272,3706,6110,5326,3752,4676,5327,4273,5403,4767,5631,6500,5699,5880,3475,5039, -6294,5562,5125,4348,4301,4482,4068,5126,4593,5700,3380,3462,5981,5563,3824,5404, -4970,5511,3825,4738,6295,6501,5452,4516,6111,5881,5564,6502,6296,5982,6503,4213, -4163,3454,6504,6112,4009,4450,6113,4658,6297,6114,3035,6505,6115,3995,4904,4739, -4563,4942,4110,5040,3661,3928,5362,3674,6506,5292,3612,4791,5565,4149,5983,5328, -5259,5021,4725,4577,4564,4517,4364,6298,5405,4578,5260,4594,4156,4157,5453,3592, -3491,6507,5127,5512,4709,4922,5984,5701,4726,4289,6508,4015,6116,5128,4628,3424, -4241,5779,6299,4905,6509,6510,5454,5702,5780,6300,4365,4923,3971,6511,5161,3270, -3158,5985,4100, 867,5129,5703,6117,5363,3695,3301,5513,4467,6118,6512,5455,4232, -4242,4629,6513,3959,4478,6514,5514,5329,5986,4850,5162,5566,3846,4694,6119,5456, -4869,5781,3779,6301,5704,5987,5515,4710,6302,5882,6120,4392,5364,5705,6515,6121, -6516,6517,3736,5988,5457,5989,4695,2457,5883,4551,5782,6303,6304,6305,5130,4971, -6122,5163,6123,4870,3263,5365,3150,4871,6518,6306,5783,5069,5706,3513,3498,4409, -5330,5632,5366,5458,5459,3991,5990,4502,3324,5991,5784,3696,4518,5633,4119,6519, -4630,5634,4417,5707,4832,5992,3418,6124,5993,5567,4768,5218,6520,4595,3458,5367, -6125,5635,6126,4202,6521,4740,4924,6307,3981,4069,4385,6308,3883,2675,4051,3834, -4302,4483,5568,5994,4972,4101,5368,6309,5164,5884,3922,6127,6522,6523,5261,5460, -5187,4164,5219,3538,5516,4111,3524,5995,6310,6311,5369,3181,3386,2484,5188,3464, -5569,3627,5708,6524,5406,5165,4677,4492,6312,4872,4851,5885,4468,5996,6313,5709, -5710,6128,2470,5886,6314,5293,4882,5785,3325,5461,5101,6129,5711,5786,6525,4906, -6526,6527,4418,5887,5712,4808,2907,3701,5713,5888,6528,3765,5636,5331,6529,6530, -3593,5889,3637,4943,3692,5714,5787,4925,6315,6130,5462,4405,6131,6132,6316,5262, -6531,6532,5715,3859,5716,5070,4696,5102,3929,5788,3987,4792,5997,6533,6534,3920, -4809,5000,5998,6535,2974,5370,6317,5189,5263,5717,3826,6536,3953,5001,4883,3190, -5463,5890,4973,5999,4741,6133,6134,3607,5570,6000,4711,3362,3630,4552,5041,6318, -6001,2950,2953,5637,4646,5371,4944,6002,2044,4120,3429,6319,6537,5103,4833,6538, -6539,4884,4647,3884,6003,6004,4758,3835,5220,5789,4565,5407,6540,6135,5294,4697, -4852,6320,6321,3206,4907,6541,6322,4945,6542,6136,6543,6323,6005,4631,3519,6544, -5891,6545,5464,3784,5221,6546,5571,4659,6547,6324,6137,5190,6548,3853,6549,4016, -4834,3954,6138,5332,3827,4017,3210,3546,4469,5408,5718,3505,4648,5790,5131,5638, -5791,5465,4727,4318,6325,6326,5792,4553,4010,4698,3439,4974,3638,4335,3085,6006, -5104,5042,5166,5892,5572,6327,4356,4519,5222,5573,5333,5793,5043,6550,5639,5071, -4503,6328,6139,6551,6140,3914,3901,5372,6007,5640,4728,4793,3976,3836,4885,6552, -4127,6553,4451,4102,5002,6554,3686,5105,6555,5191,5072,5295,4611,5794,5296,6556, -5893,5264,5894,4975,5466,5265,4699,4976,4370,4056,3492,5044,4886,6557,5795,4432, -4769,4357,5467,3940,4660,4290,6141,4484,4770,4661,3992,6329,4025,4662,5022,4632, -4835,4070,5297,4663,4596,5574,5132,5409,5895,6142,4504,5192,4664,5796,5896,3885, -5575,5797,5023,4810,5798,3732,5223,4712,5298,4084,5334,5468,6143,4052,4053,4336, -4977,4794,6558,5335,4908,5576,5224,4233,5024,4128,5469,5225,4873,6008,5045,4729, -4742,4633,3675,4597,6559,5897,5133,5577,5003,5641,5719,6330,6560,3017,2382,3854, -4406,4811,6331,4393,3964,4946,6561,2420,3722,6562,4926,4378,3247,1736,4442,6332, -5134,6333,5226,3996,2918,5470,4319,4003,4598,4743,4744,4485,3785,3902,5167,5004, -5373,4394,5898,6144,4874,1793,3997,6334,4085,4214,5106,5642,4909,5799,6009,4419, -4189,3330,5899,4165,4420,5299,5720,5227,3347,6145,4081,6335,2876,3930,6146,3293, -3786,3910,3998,5900,5300,5578,2840,6563,5901,5579,6147,3531,5374,6564,6565,5580, -4759,5375,6566,6148,3559,5643,6336,6010,5517,6337,6338,5721,5902,3873,6011,6339, -6567,5518,3868,3649,5722,6568,4771,4947,6569,6149,4812,6570,2853,5471,6340,6341, -5644,4795,6342,6012,5723,6343,5724,6013,4349,6344,3160,6150,5193,4599,4514,4493, -5168,4320,6345,4927,3666,4745,5169,5903,5005,4928,6346,5725,6014,4730,4203,5046, -4948,3395,5170,6015,4150,6016,5726,5519,6347,5047,3550,6151,6348,4197,4310,5904, -6571,5581,2965,6152,4978,3960,4291,5135,6572,5301,5727,4129,4026,5905,4853,5728, -5472,6153,6349,4533,2700,4505,5336,4678,3583,5073,2994,4486,3043,4554,5520,6350, -6017,5800,4487,6351,3931,4103,5376,6352,4011,4321,4311,4190,5136,6018,3988,3233, -4350,5906,5645,4198,6573,5107,3432,4191,3435,5582,6574,4139,5410,6353,5411,3944, -5583,5074,3198,6575,6354,4358,6576,5302,4600,5584,5194,5412,6577,6578,5585,5413, -5303,4248,5414,3879,4433,6579,4479,5025,4854,5415,6355,4760,4772,3683,2978,4700, -3797,4452,3965,3932,3721,4910,5801,6580,5195,3551,5907,3221,3471,3029,6019,3999, -5908,5909,5266,5267,3444,3023,3828,3170,4796,5646,4979,4259,6356,5647,5337,3694, -6357,5648,5338,4520,4322,5802,3031,3759,4071,6020,5586,4836,4386,5048,6581,3571, -4679,4174,4949,6154,4813,3787,3402,3822,3958,3215,3552,5268,4387,3933,4950,4359, -6021,5910,5075,3579,6358,4234,4566,5521,6359,3613,5049,6022,5911,3375,3702,3178, -4911,5339,4521,6582,6583,4395,3087,3811,5377,6023,6360,6155,4027,5171,5649,4421, -4249,2804,6584,2270,6585,4000,4235,3045,6156,5137,5729,4140,4312,3886,6361,4330, -6157,4215,6158,3500,3676,4929,4331,3713,4930,5912,4265,3776,3368,5587,4470,4855, -3038,4980,3631,6159,6160,4132,4680,6161,6362,3923,4379,5588,4255,6586,4121,6587, -6363,4649,6364,3288,4773,4774,6162,6024,6365,3543,6588,4274,3107,3737,5050,5803, -4797,4522,5589,5051,5730,3714,4887,5378,4001,4523,6163,5026,5522,4701,4175,2791, -3760,6589,5473,4224,4133,3847,4814,4815,4775,3259,5416,6590,2738,6164,6025,5304, -3733,5076,5650,4816,5590,6591,6165,6592,3934,5269,6593,3396,5340,6594,5804,3445, -3602,4042,4488,5731,5732,3525,5591,4601,5196,6166,6026,5172,3642,4612,3202,4506, -4798,6366,3818,5108,4303,5138,5139,4776,3332,4304,2915,3415,4434,5077,5109,4856, -2879,5305,4817,6595,5913,3104,3144,3903,4634,5341,3133,5110,5651,5805,6167,4057, -5592,2945,4371,5593,6596,3474,4182,6367,6597,6168,4507,4279,6598,2822,6599,4777, -4713,5594,3829,6169,3887,5417,6170,3653,5474,6368,4216,2971,5228,3790,4579,6369, -5733,6600,6601,4951,4746,4555,6602,5418,5475,6027,3400,4665,5806,6171,4799,6028, -5052,6172,3343,4800,4747,5006,6370,4556,4217,5476,4396,5229,5379,5477,3839,5914, -5652,5807,4714,3068,4635,5808,6173,5342,4192,5078,5419,5523,5734,6174,4557,6175, -4602,6371,6176,6603,5809,6372,5735,4260,3869,5111,5230,6029,5112,6177,3126,4681, -5524,5915,2706,3563,4748,3130,6178,4018,5525,6604,6605,5478,4012,4837,6606,4534, -4193,5810,4857,3615,5479,6030,4082,3697,3539,4086,5270,3662,4508,4931,5916,4912, -5811,5027,3888,6607,4397,3527,3302,3798,2775,2921,2637,3966,4122,4388,4028,4054, -1633,4858,5079,3024,5007,3982,3412,5736,6608,3426,3236,5595,3030,6179,3427,3336, -3279,3110,6373,3874,3039,5080,5917,5140,4489,3119,6374,5812,3405,4494,6031,4666, -4141,6180,4166,6032,5813,4981,6609,5081,4422,4982,4112,3915,5653,3296,3983,6375, -4266,4410,5654,6610,6181,3436,5082,6611,5380,6033,3819,5596,4535,5231,5306,5113, -6612,4952,5918,4275,3113,6613,6376,6182,6183,5814,3073,4731,4838,5008,3831,6614, -4888,3090,3848,4280,5526,5232,3014,5655,5009,5737,5420,5527,6615,5815,5343,5173, -5381,4818,6616,3151,4953,6617,5738,2796,3204,4360,2989,4281,5739,5174,5421,5197, -3132,5141,3849,5142,5528,5083,3799,3904,4839,5480,2880,4495,3448,6377,6184,5271, -5919,3771,3193,6034,6035,5920,5010,6036,5597,6037,6378,6038,3106,5422,6618,5423, -5424,4142,6619,4889,5084,4890,4313,5740,6620,3437,5175,5307,5816,4199,5198,5529, -5817,5199,5656,4913,5028,5344,3850,6185,2955,5272,5011,5818,4567,4580,5029,5921, -3616,5233,6621,6622,6186,4176,6039,6379,6380,3352,5200,5273,2908,5598,5234,3837, -5308,6623,6624,5819,4496,4323,5309,5201,6625,6626,4983,3194,3838,4167,5530,5922, -5274,6381,6382,3860,3861,5599,3333,4292,4509,6383,3553,5481,5820,5531,4778,6187, -3955,3956,4324,4389,4218,3945,4325,3397,2681,5923,4779,5085,4019,5482,4891,5382, -5383,6040,4682,3425,5275,4094,6627,5310,3015,5483,5657,4398,5924,3168,4819,6628, -5925,6629,5532,4932,4613,6041,6630,4636,6384,4780,4204,5658,4423,5821,3989,4683, -5822,6385,4954,6631,5345,6188,5425,5012,5384,3894,6386,4490,4104,6632,5741,5053, -6633,5823,5926,5659,5660,5927,6634,5235,5742,5824,4840,4933,4820,6387,4859,5928, -4955,6388,4143,3584,5825,5346,5013,6635,5661,6389,5014,5484,5743,4337,5176,5662, -6390,2836,6391,3268,6392,6636,6042,5236,6637,4158,6638,5744,5663,4471,5347,3663, -4123,5143,4293,3895,6639,6640,5311,5929,5826,3800,6189,6393,6190,5664,5348,3554, -3594,4749,4603,6641,5385,4801,6043,5827,4183,6642,5312,5426,4761,6394,5665,6191, -4715,2669,6643,6644,5533,3185,5427,5086,5930,5931,5386,6192,6044,6645,4781,4013, -5745,4282,4435,5534,4390,4267,6045,5746,4984,6046,2743,6193,3501,4087,5485,5932, -5428,4184,4095,5747,4061,5054,3058,3862,5933,5600,6646,5144,3618,6395,3131,5055, -5313,6396,4650,4956,3855,6194,3896,5202,4985,4029,4225,6195,6647,5828,5486,5829, -3589,3002,6648,6397,4782,5276,6649,6196,6650,4105,3803,4043,5237,5830,6398,4096, -3643,6399,3528,6651,4453,3315,4637,6652,3984,6197,5535,3182,3339,6653,3096,2660, -6400,6654,3449,5934,4250,4236,6047,6401,5831,6655,5487,3753,4062,5832,6198,6199, -6656,3766,6657,3403,4667,6048,6658,4338,2897,5833,3880,2797,3780,4326,6659,5748, -5015,6660,5387,4351,5601,4411,6661,3654,4424,5935,4339,4072,5277,4568,5536,6402, -6662,5238,6663,5349,5203,6200,5204,6201,5145,4536,5016,5056,4762,5834,4399,4957, -6202,6403,5666,5749,6664,4340,6665,5936,5177,5667,6666,6667,3459,4668,6404,6668, -6669,4543,6203,6670,4276,6405,4480,5537,6671,4614,5205,5668,6672,3348,2193,4763, -6406,6204,5937,5602,4177,5669,3419,6673,4020,6205,4443,4569,5388,3715,3639,6407, -6049,4058,6206,6674,5938,4544,6050,4185,4294,4841,4651,4615,5488,6207,6408,6051, -5178,3241,3509,5835,6208,4958,5836,4341,5489,5278,6209,2823,5538,5350,5206,5429, -6675,4638,4875,4073,3516,4684,4914,4860,5939,5603,5389,6052,5057,3237,5490,3791, -6676,6409,6677,4821,4915,4106,5351,5058,4243,5539,4244,5604,4842,4916,5239,3028, -3716,5837,5114,5605,5390,5940,5430,6210,4332,6678,5540,4732,3667,3840,6053,4305, -3408,5670,5541,6410,2744,5240,5750,6679,3234,5606,6680,5607,5671,3608,4283,4159, -4400,5352,4783,6681,6411,6682,4491,4802,6211,6412,5941,6413,6414,5542,5751,6683, -4669,3734,5942,6684,6415,5943,5059,3328,4670,4144,4268,6685,6686,6687,6688,4372, -3603,6689,5944,5491,4373,3440,6416,5543,4784,4822,5608,3792,4616,5838,5672,3514, -5391,6417,4892,6690,4639,6691,6054,5673,5839,6055,6692,6056,5392,6212,4038,5544, -5674,4497,6057,6693,5840,4284,5675,4021,4545,5609,6418,4454,6419,6213,4113,4472, -5314,3738,5087,5279,4074,5610,4959,4063,3179,4750,6058,6420,6214,3476,4498,4716, -5431,4960,4685,6215,5241,6694,6421,6216,6695,5841,5945,6422,3748,5946,5179,3905, -5752,5545,5947,4374,6217,4455,6423,4412,6218,4803,5353,6696,3832,5280,6219,4327, -4702,6220,6221,6059,4652,5432,6424,3749,4751,6425,5753,4986,5393,4917,5948,5030, -5754,4861,4733,6426,4703,6697,6222,4671,5949,4546,4961,5180,6223,5031,3316,5281, -6698,4862,4295,4934,5207,3644,6427,5842,5950,6428,6429,4570,5843,5282,6430,6224, -5088,3239,6060,6699,5844,5755,6061,6431,2701,5546,6432,5115,5676,4039,3993,3327, -4752,4425,5315,6433,3941,6434,5677,4617,4604,3074,4581,6225,5433,6435,6226,6062, -4823,5756,5116,6227,3717,5678,4717,5845,6436,5679,5846,6063,5847,6064,3977,3354, -6437,3863,5117,6228,5547,5394,4499,4524,6229,4605,6230,4306,4500,6700,5951,6065, -3693,5952,5089,4366,4918,6701,6231,5548,6232,6702,6438,4704,5434,6703,6704,5953, -4168,6705,5680,3420,6706,5242,4407,6066,3812,5757,5090,5954,4672,4525,3481,5681, -4618,5395,5354,5316,5955,6439,4962,6707,4526,6440,3465,4673,6067,6441,5682,6708, -5435,5492,5758,5683,4619,4571,4674,4804,4893,4686,5493,4753,6233,6068,4269,6442, -6234,5032,4705,5146,5243,5208,5848,6235,6443,4963,5033,4640,4226,6236,5849,3387, -6444,6445,4436,4437,5850,4843,5494,4785,4894,6709,4361,6710,5091,5956,3331,6237, -4987,5549,6069,6711,4342,3517,4473,5317,6070,6712,6071,4706,6446,5017,5355,6713, -6714,4988,5436,6447,4734,5759,6715,4735,4547,4456,4754,6448,5851,6449,6450,3547, -5852,5318,6451,6452,5092,4205,6716,6238,4620,4219,5611,6239,6072,4481,5760,5957, -5958,4059,6240,6453,4227,4537,6241,5761,4030,4186,5244,5209,3761,4457,4876,3337, -5495,5181,6242,5959,5319,5612,5684,5853,3493,5854,6073,4169,5613,5147,4895,6074, -5210,6717,5182,6718,3830,6243,2798,3841,6075,6244,5855,5614,3604,4606,5496,5685, -5118,5356,6719,6454,5960,5357,5961,6720,4145,3935,4621,5119,5962,4261,6721,6455, -4786,5963,4375,4582,6245,6246,6247,6076,5437,4877,5856,3376,4380,6248,4160,6722, -5148,6456,5211,6457,6723,4718,6458,6724,6249,5358,4044,3297,6459,6250,5857,5615, -5497,5245,6460,5498,6725,6251,6252,5550,3793,5499,2959,5396,6461,6462,4572,5093, -5500,5964,3806,4146,6463,4426,5762,5858,6077,6253,4755,3967,4220,5965,6254,4989, -5501,6464,4352,6726,6078,4764,2290,5246,3906,5438,5283,3767,4964,2861,5763,5094, -6255,6256,4622,5616,5859,5860,4707,6727,4285,4708,4824,5617,6257,5551,4787,5212, -4965,4935,4687,6465,6728,6466,5686,6079,3494,4413,2995,5247,5966,5618,6729,5967, -5764,5765,5687,5502,6730,6731,6080,5397,6467,4990,6258,6732,4538,5060,5619,6733, -4719,5688,5439,5018,5149,5284,5503,6734,6081,4607,6259,5120,3645,5861,4583,6260, -4584,4675,5620,4098,5440,6261,4863,2379,3306,4585,5552,5689,4586,5285,6735,4864, -6736,5286,6082,6737,4623,3010,4788,4381,4558,5621,4587,4896,3698,3161,5248,4353, -4045,6262,3754,5183,4588,6738,6263,6739,6740,5622,3936,6741,6468,6742,6264,5095, -6469,4991,5968,6743,4992,6744,6083,4897,6745,4256,5766,4307,3108,3968,4444,5287, -3889,4343,6084,4510,6085,4559,6086,4898,5969,6746,5623,5061,4919,5249,5250,5504, -5441,6265,5320,4878,3242,5862,5251,3428,6087,6747,4237,5624,5442,6266,5553,4539, -6748,2585,3533,5398,4262,6088,5150,4736,4438,6089,6267,5505,4966,6749,6268,6750, -6269,5288,5554,3650,6090,6091,4624,6092,5690,6751,5863,4270,5691,4277,5555,5864, -6752,5692,4720,4865,6470,5151,4688,4825,6753,3094,6754,6471,3235,4653,6755,5213, -5399,6756,3201,4589,5865,4967,6472,5866,6473,5019,3016,6757,5321,4756,3957,4573, -6093,4993,5767,4721,6474,6758,5625,6759,4458,6475,6270,6760,5556,4994,5214,5252, -6271,3875,5768,6094,5034,5506,4376,5769,6761,2120,6476,5253,5770,6762,5771,5970, -3990,5971,5557,5558,5772,6477,6095,2787,4641,5972,5121,6096,6097,6272,6763,3703, -5867,5507,6273,4206,6274,4789,6098,6764,3619,3646,3833,3804,2394,3788,4936,3978, -4866,4899,6099,6100,5559,6478,6765,3599,5868,6101,5869,5870,6275,6766,4527,6767, -*******************************************************************************/ -}; - - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/JISFreq.tab b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/JISFreq.tab deleted file mode 100644 index 715321f..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/JISFreq.tab +++ /dev/null @@ -1,589 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -//Sampling from about 20M text materials include literature and computer technology - -// Japanese frequency table, applied to both S-JIS and EUC-JP -//They are sorted in order. - -/****************************************************************************** - * 128 --> 0.77094 - * 256 --> 0.85710 - * 512 --> 0.92635 - * 1024 --> 0.97130 - * 2048 --> 0.99431 - * - * Idea Distribution Ratio = 0.92635 / (1-0.92635) = 12.58 - * Random Distribution Ration = 512 / (2965+62+83+86-512) = 0.191 - * - * Typical Distribution Ratio, 25% of IDR - *****************************************************************************/ - -#define JIS_TYPICAL_DISTRIBUTION_RATIO (float) 3.0 - - -//Char to FreqOrder table , -#define JIS_TABLE_SIZE 4368 - -static const PRInt16 JISCharToFreqOrder[] = -{ - 40, 1, 6, 182, 152, 180, 295,2127, 285, 381,3295,4304,3068,4606,3165,3510, // 16 -3511,1822,2785,4607,1193,2226,5070,4608, 171,2996,1247, 18, 179,5071, 856,1661, // 32 -1262,5072, 619, 127,3431,3512,3230,1899,1700, 232, 228,1294,1298, 284, 283,2041, // 48 -2042,1061,1062, 48, 49, 44, 45, 433, 434,1040,1041, 996, 787,2997,1255,4305, // 64 -2108,4609,1684,1648,5073,5074,5075,5076,5077,5078,3687,5079,4610,5080,3927,3928, // 80 -5081,3296,3432, 290,2285,1471,2187,5082,2580,2825,1303,2140,1739,1445,2691,3375, // 96 -1691,3297,4306,4307,4611, 452,3376,1182,2713,3688,3069,4308,5083,5084,5085,5086, // 112 -5087,5088,5089,5090,5091,5092,5093,5094,5095,5096,5097,5098,5099,5100,5101,5102, // 128 -5103,5104,5105,5106,5107,5108,5109,5110,5111,5112,4097,5113,5114,5115,5116,5117, // 144 -5118,5119,5120,5121,5122,5123,5124,5125,5126,5127,5128,5129,5130,5131,5132,5133, // 160 -5134,5135,5136,5137,5138,5139,5140,5141,5142,5143,5144,5145,5146,5147,5148,5149, // 176 -5150,5151,5152,4612,5153,5154,5155,5156,5157,5158,5159,5160,5161,5162,5163,5164, // 192 -5165,5166,5167,5168,5169,5170,5171,5172,5173,5174,5175,1472, 598, 618, 820,1205, // 208 -1309,1412,1858,1307,1692,5176,5177,5178,5179,5180,5181,5182,1142,1452,1234,1172, // 224 -1875,2043,2149,1793,1382,2973, 925,2404,1067,1241, 960,1377,2935,1491, 919,1217, // 240 -1865,2030,1406,1499,2749,4098,5183,5184,5185,5186,5187,5188,2561,4099,3117,1804, // 256 -2049,3689,4309,3513,1663,5189,3166,3118,3298,1587,1561,3433,5190,3119,1625,2998, // 272 -3299,4613,1766,3690,2786,4614,5191,5192,5193,5194,2161, 26,3377, 2,3929, 20, // 288 -3691, 47,4100, 50, 17, 16, 35, 268, 27, 243, 42, 155, 24, 154, 29, 184, // 304 - 4, 91, 14, 92, 53, 396, 33, 289, 9, 37, 64, 620, 21, 39, 321, 5, // 320 - 12, 11, 52, 13, 3, 208, 138, 0, 7, 60, 526, 141, 151,1069, 181, 275, // 336 -1591, 83, 132,1475, 126, 331, 829, 15, 69, 160, 59, 22, 157, 55,1079, 312, // 352 - 109, 38, 23, 25, 10, 19, 79,5195, 61, 382,1124, 8, 30,5196,5197,5198, // 368 -5199,5200,5201,5202,5203,5204,5205,5206, 89, 62, 74, 34,2416, 112, 139, 196, // 384 - 271, 149, 84, 607, 131, 765, 46, 88, 153, 683, 76, 874, 101, 258, 57, 80, // 400 - 32, 364, 121,1508, 169,1547, 68, 235, 145,2999, 41, 360,3027, 70, 63, 31, // 416 - 43, 259, 262,1383, 99, 533, 194, 66, 93, 846, 217, 192, 56, 106, 58, 565, // 432 - 280, 272, 311, 256, 146, 82, 308, 71, 100, 128, 214, 655, 110, 261, 104,1140, // 448 - 54, 51, 36, 87, 67,3070, 185,2618,2936,2020, 28,1066,2390,2059,5207,5208, // 464 -5209,5210,5211,5212,5213,5214,5215,5216,4615,5217,5218,5219,5220,5221,5222,5223, // 480 -5224,5225,5226,5227,5228,5229,5230,5231,5232,5233,5234,5235,5236,3514,5237,5238, // 496 -5239,5240,5241,5242,5243,5244,2297,2031,4616,4310,3692,5245,3071,5246,3598,5247, // 512 -4617,3231,3515,5248,4101,4311,4618,3808,4312,4102,5249,4103,4104,3599,5250,5251, // 528 -5252,5253,5254,5255,5256,5257,5258,5259,5260,5261,5262,5263,5264,5265,5266,5267, // 544 -5268,5269,5270,5271,5272,5273,5274,5275,5276,5277,5278,5279,5280,5281,5282,5283, // 560 -5284,5285,5286,5287,5288,5289,5290,5291,5292,5293,5294,5295,5296,5297,5298,5299, // 576 -5300,5301,5302,5303,5304,5305,5306,5307,5308,5309,5310,5311,5312,5313,5314,5315, // 592 -5316,5317,5318,5319,5320,5321,5322,5323,5324,5325,5326,5327,5328,5329,5330,5331, // 608 -5332,5333,5334,5335,5336,5337,5338,5339,5340,5341,5342,5343,5344,5345,5346,5347, // 624 -5348,5349,5350,5351,5352,5353,5354,5355,5356,5357,5358,5359,5360,5361,5362,5363, // 640 -5364,5365,5366,5367,5368,5369,5370,5371,5372,5373,5374,5375,5376,5377,5378,5379, // 656 -5380,5381, 363, 642,2787,2878,2788,2789,2316,3232,2317,3434,2011, 165,1942,3930, // 672 -3931,3932,3933,5382,4619,5383,4620,5384,5385,5386,5387,5388,5389,5390,5391,5392, // 688 -5393,5394,5395,5396,5397,5398,5399,5400,5401,5402,5403,5404,5405,5406,5407,5408, // 704 -5409,5410,5411,5412,5413,5414,5415,5416,5417,5418,5419,5420,5421,5422,5423,5424, // 720 -5425,5426,5427,5428,5429,5430,5431,5432,5433,5434,5435,5436,5437,5438,5439,5440, // 736 -5441,5442,5443,5444,5445,5446,5447,5448,5449,5450,5451,5452,5453,5454,5455,5456, // 752 -5457,5458,5459,5460,5461,5462,5463,5464,5465,5466,5467,5468,5469,5470,5471,5472, // 768 -5473,5474,5475,5476,5477,5478,5479,5480,5481,5482,5483,5484,5485,5486,5487,5488, // 784 -5489,5490,5491,5492,5493,5494,5495,5496,5497,5498,5499,5500,5501,5502,5503,5504, // 800 -5505,5506,5507,5508,5509,5510,5511,5512,5513,5514,5515,5516,5517,5518,5519,5520, // 816 -5521,5522,5523,5524,5525,5526,5527,5528,5529,5530,5531,5532,5533,5534,5535,5536, // 832 -5537,5538,5539,5540,5541,5542,5543,5544,5545,5546,5547,5548,5549,5550,5551,5552, // 848 -5553,5554,5555,5556,5557,5558,5559,5560,5561,5562,5563,5564,5565,5566,5567,5568, // 864 -5569,5570,5571,5572,5573,5574,5575,5576,5577,5578,5579,5580,5581,5582,5583,5584, // 880 -5585,5586,5587,5588,5589,5590,5591,5592,5593,5594,5595,5596,5597,5598,5599,5600, // 896 -5601,5602,5603,5604,5605,5606,5607,5608,5609,5610,5611,5612,5613,5614,5615,5616, // 912 -5617,5618,5619,5620,5621,5622,5623,5624,5625,5626,5627,5628,5629,5630,5631,5632, // 928 -5633,5634,5635,5636,5637,5638,5639,5640,5641,5642,5643,5644,5645,5646,5647,5648, // 944 -5649,5650,5651,5652,5653,5654,5655,5656,5657,5658,5659,5660,5661,5662,5663,5664, // 960 -5665,5666,5667,5668,5669,5670,5671,5672,5673,5674,5675,5676,5677,5678,5679,5680, // 976 -5681,5682,5683,5684,5685,5686,5687,5688,5689,5690,5691,5692,5693,5694,5695,5696, // 992 -5697,5698,5699,5700,5701,5702,5703,5704,5705,5706,5707,5708,5709,5710,5711,5712, // 1008 -5713,5714,5715,5716,5717,5718,5719,5720,5721,5722,5723,5724,5725,5726,5727,5728, // 1024 -5729,5730,5731,5732,5733,5734,5735,5736,5737,5738,5739,5740,5741,5742,5743,5744, // 1040 -5745,5746,5747,5748,5749,5750,5751,5752,5753,5754,5755,5756,5757,5758,5759,5760, // 1056 -5761,5762,5763,5764,5765,5766,5767,5768,5769,5770,5771,5772,5773,5774,5775,5776, // 1072 -5777,5778,5779,5780,5781,5782,5783,5784,5785,5786,5787,5788,5789,5790,5791,5792, // 1088 -5793,5794,5795,5796,5797,5798,5799,5800,5801,5802,5803,5804,5805,5806,5807,5808, // 1104 -5809,5810,5811,5812,5813,5814,5815,5816,5817,5818,5819,5820,5821,5822,5823,5824, // 1120 -5825,5826,5827,5828,5829,5830,5831,5832,5833,5834,5835,5836,5837,5838,5839,5840, // 1136 -5841,5842,5843,5844,5845,5846,5847,5848,5849,5850,5851,5852,5853,5854,5855,5856, // 1152 -5857,5858,5859,5860,5861,5862,5863,5864,5865,5866,5867,5868,5869,5870,5871,5872, // 1168 -5873,5874,5875,5876,5877,5878,5879,5880,5881,5882,5883,5884,5885,5886,5887,5888, // 1184 -5889,5890,5891,5892,5893,5894,5895,5896,5897,5898,5899,5900,5901,5902,5903,5904, // 1200 -5905,5906,5907,5908,5909,5910,5911,5912,5913,5914,5915,5916,5917,5918,5919,5920, // 1216 -5921,5922,5923,5924,5925,5926,5927,5928,5929,5930,5931,5932,5933,5934,5935,5936, // 1232 -5937,5938,5939,5940,5941,5942,5943,5944,5945,5946,5947,5948,5949,5950,5951,5952, // 1248 -5953,5954,5955,5956,5957,5958,5959,5960,5961,5962,5963,5964,5965,5966,5967,5968, // 1264 -5969,5970,5971,5972,5973,5974,5975,5976,5977,5978,5979,5980,5981,5982,5983,5984, // 1280 -5985,5986,5987,5988,5989,5990,5991,5992,5993,5994,5995,5996,5997,5998,5999,6000, // 1296 -6001,6002,6003,6004,6005,6006,6007,6008,6009,6010,6011,6012,6013,6014,6015,6016, // 1312 -6017,6018,6019,6020,6021,6022,6023,6024,6025,6026,6027,6028,6029,6030,6031,6032, // 1328 -6033,6034,6035,6036,6037,6038,6039,6040,6041,6042,6043,6044,6045,6046,6047,6048, // 1344 -6049,6050,6051,6052,6053,6054,6055,6056,6057,6058,6059,6060,6061,6062,6063,6064, // 1360 -6065,6066,6067,6068,6069,6070,6071,6072,6073,6074,6075,6076,6077,6078,6079,6080, // 1376 -6081,6082,6083,6084,6085,6086,6087,6088,6089,6090,6091,6092,6093,6094,6095,6096, // 1392 -6097,6098,6099,6100,6101,6102,6103,6104,6105,6106,6107,6108,6109,6110,6111,6112, // 1408 -6113,6114,2044,2060,4621, 997,1235, 473,1186,4622, 920,3378,6115,6116, 379,1108, // 1424 -4313,2657,2735,3934,6117,3809, 636,3233, 573,1026,3693,3435,2974,3300,2298,4105, // 1440 - 854,2937,2463, 393,2581,2417, 539, 752,1280,2750,2480, 140,1161, 440, 708,1569, // 1456 - 665,2497,1746,1291,1523,3000, 164,1603, 847,1331, 537,1997, 486, 508,1693,2418, // 1472 -1970,2227, 878,1220, 299,1030, 969, 652,2751, 624,1137,3301,2619, 65,3302,2045, // 1488 -1761,1859,3120,1930,3694,3516, 663,1767, 852, 835,3695, 269, 767,2826,2339,1305, // 1504 - 896,1150, 770,1616,6118, 506,1502,2075,1012,2519, 775,2520,2975,2340,2938,4314, // 1520 -3028,2086,1224,1943,2286,6119,3072,4315,2240,1273,1987,3935,1557, 175, 597, 985, // 1536 -3517,2419,2521,1416,3029, 585, 938,1931,1007,1052,1932,1685,6120,3379,4316,4623, // 1552 - 804, 599,3121,1333,2128,2539,1159,1554,2032,3810, 687,2033,2904, 952, 675,1467, // 1568 -3436,6121,2241,1096,1786,2440,1543,1924, 980,1813,2228, 781,2692,1879, 728,1918, // 1584 -3696,4624, 548,1950,4625,1809,1088,1356,3303,2522,1944, 502, 972, 373, 513,2827, // 1600 - 586,2377,2391,1003,1976,1631,6122,2464,1084, 648,1776,4626,2141, 324, 962,2012, // 1616 -2177,2076,1384, 742,2178,1448,1173,1810, 222, 102, 301, 445, 125,2420, 662,2498, // 1632 - 277, 200,1476,1165,1068, 224,2562,1378,1446, 450,1880, 659, 791, 582,4627,2939, // 1648 -3936,1516,1274, 555,2099,3697,1020,1389,1526,3380,1762,1723,1787,2229, 412,2114, // 1664 -1900,2392,3518, 512,2597, 427,1925,2341,3122,1653,1686,2465,2499, 697, 330, 273, // 1680 - 380,2162, 951, 832, 780, 991,1301,3073, 965,2270,3519, 668,2523,2636,1286, 535, // 1696 -1407, 518, 671, 957,2658,2378, 267, 611,2197,3030,6123, 248,2299, 967,1799,2356, // 1712 - 850,1418,3437,1876,1256,1480,2828,1718,6124,6125,1755,1664,2405,6126,4628,2879, // 1728 -2829, 499,2179, 676,4629, 557,2329,2214,2090, 325,3234, 464, 811,3001, 992,2342, // 1744 -2481,1232,1469, 303,2242, 466,1070,2163, 603,1777,2091,4630,2752,4631,2714, 322, // 1760 -2659,1964,1768, 481,2188,1463,2330,2857,3600,2092,3031,2421,4632,2318,2070,1849, // 1776 -2598,4633,1302,2254,1668,1701,2422,3811,2905,3032,3123,2046,4106,1763,1694,4634, // 1792 -1604, 943,1724,1454, 917, 868,2215,1169,2940, 552,1145,1800,1228,1823,1955, 316, // 1808 -1080,2510, 361,1807,2830,4107,2660,3381,1346,1423,1134,4108,6127, 541,1263,1229, // 1824 -1148,2540, 545, 465,1833,2880,3438,1901,3074,2482, 816,3937, 713,1788,2500, 122, // 1840 -1575, 195,1451,2501,1111,6128, 859, 374,1225,2243,2483,4317, 390,1033,3439,3075, // 1856 -2524,1687, 266, 793,1440,2599, 946, 779, 802, 507, 897,1081, 528,2189,1292, 711, // 1872 -1866,1725,1167,1640, 753, 398,2661,1053, 246, 348,4318, 137,1024,3440,1600,2077, // 1888 -2129, 825,4319, 698, 238, 521, 187,2300,1157,2423,1641,1605,1464,1610,1097,2541, // 1904 -1260,1436, 759,2255,1814,2150, 705,3235, 409,2563,3304, 561,3033,2005,2564, 726, // 1920 -1956,2343,3698,4109, 949,3812,3813,3520,1669, 653,1379,2525, 881,2198, 632,2256, // 1936 -1027, 778,1074, 733,1957, 514,1481,2466, 554,2180, 702,3938,1606,1017,1398,6129, // 1952 -1380,3521, 921, 993,1313, 594, 449,1489,1617,1166, 768,1426,1360, 495,1794,3601, // 1968 -1177,3602,1170,4320,2344, 476, 425,3167,4635,3168,1424, 401,2662,1171,3382,1998, // 1984 -1089,4110, 477,3169, 474,6130,1909, 596,2831,1842, 494, 693,1051,1028,1207,3076, // 2000 - 606,2115, 727,2790,1473,1115, 743,3522, 630, 805,1532,4321,2021, 366,1057, 838, // 2016 - 684,1114,2142,4322,2050,1492,1892,1808,2271,3814,2424,1971,1447,1373,3305,1090, // 2032 -1536,3939,3523,3306,1455,2199, 336, 369,2331,1035, 584,2393, 902, 718,2600,6131, // 2048 -2753, 463,2151,1149,1611,2467, 715,1308,3124,1268, 343,1413,3236,1517,1347,2663, // 2064 -2093,3940,2022,1131,1553,2100,2941,1427,3441,2942,1323,2484,6132,1980, 872,2368, // 2080 -2441,2943, 320,2369,2116,1082, 679,1933,3941,2791,3815, 625,1143,2023, 422,2200, // 2096 -3816,6133, 730,1695, 356,2257,1626,2301,2858,2637,1627,1778, 937, 883,2906,2693, // 2112 -3002,1769,1086, 400,1063,1325,3307,2792,4111,3077, 456,2345,1046, 747,6134,1524, // 2128 - 884,1094,3383,1474,2164,1059, 974,1688,2181,2258,1047, 345,1665,1187, 358, 875, // 2144 -3170, 305, 660,3524,2190,1334,1135,3171,1540,1649,2542,1527, 927, 968,2793, 885, // 2160 -1972,1850, 482, 500,2638,1218,1109,1085,2543,1654,2034, 876, 78,2287,1482,1277, // 2176 - 861,1675,1083,1779, 724,2754, 454, 397,1132,1612,2332, 893, 672,1237, 257,2259, // 2192 -2370, 135,3384, 337,2244, 547, 352, 340, 709,2485,1400, 788,1138,2511, 540, 772, // 2208 -1682,2260,2272,2544,2013,1843,1902,4636,1999,1562,2288,4637,2201,1403,1533, 407, // 2224 - 576,3308,1254,2071, 978,3385, 170, 136,1201,3125,2664,3172,2394, 213, 912, 873, // 2240 -3603,1713,2202, 699,3604,3699, 813,3442, 493, 531,1054, 468,2907,1483, 304, 281, // 2256 -4112,1726,1252,2094, 339,2319,2130,2639, 756,1563,2944, 748, 571,2976,1588,2425, // 2272 -2715,1851,1460,2426,1528,1392,1973,3237, 288,3309, 685,3386, 296, 892,2716,2216, // 2288 -1570,2245, 722,1747,2217, 905,3238,1103,6135,1893,1441,1965, 251,1805,2371,3700, // 2304 -2601,1919,1078, 75,2182,1509,1592,1270,2640,4638,2152,6136,3310,3817, 524, 706, // 2320 -1075, 292,3818,1756,2602, 317, 98,3173,3605,3525,1844,2218,3819,2502, 814, 567, // 2336 - 385,2908,1534,6137, 534,1642,3239, 797,6138,1670,1529, 953,4323, 188,1071, 538, // 2352 - 178, 729,3240,2109,1226,1374,2000,2357,2977, 731,2468,1116,2014,2051,6139,1261, // 2368 -1593, 803,2859,2736,3443, 556, 682, 823,1541,6140,1369,2289,1706,2794, 845, 462, // 2384 -2603,2665,1361, 387, 162,2358,1740, 739,1770,1720,1304,1401,3241,1049, 627,1571, // 2400 -2427,3526,1877,3942,1852,1500, 431,1910,1503, 677, 297,2795, 286,1433,1038,1198, // 2416 -2290,1133,1596,4113,4639,2469,1510,1484,3943,6141,2442, 108, 712,4640,2372, 866, // 2432 -3701,2755,3242,1348, 834,1945,1408,3527,2395,3243,1811, 824, 994,1179,2110,1548, // 2448 -1453, 790,3003, 690,4324,4325,2832,2909,3820,1860,3821, 225,1748, 310, 346,1780, // 2464 -2470, 821,1993,2717,2796, 828, 877,3528,2860,2471,1702,2165,2910,2486,1789, 453, // 2480 - 359,2291,1676, 73,1164,1461,1127,3311, 421, 604, 314,1037, 589, 116,2487, 737, // 2496 - 837,1180, 111, 244, 735,6142,2261,1861,1362, 986, 523, 418, 581,2666,3822, 103, // 2512 - 855, 503,1414,1867,2488,1091, 657,1597, 979, 605,1316,4641,1021,2443,2078,2001, // 2528 -1209, 96, 587,2166,1032, 260,1072,2153, 173, 94, 226,3244, 819,2006,4642,4114, // 2544 -2203, 231,1744, 782, 97,2667, 786,3387, 887, 391, 442,2219,4326,1425,6143,2694, // 2560 - 633,1544,1202, 483,2015, 592,2052,1958,2472,1655, 419, 129,4327,3444,3312,1714, // 2576 -1257,3078,4328,1518,1098, 865,1310,1019,1885,1512,1734, 469,2444, 148, 773, 436, // 2592 -1815,1868,1128,1055,4329,1245,2756,3445,2154,1934,1039,4643, 579,1238, 932,2320, // 2608 - 353, 205, 801, 115,2428, 944,2321,1881, 399,2565,1211, 678, 766,3944, 335,2101, // 2624 -1459,1781,1402,3945,2737,2131,1010, 844, 981,1326,1013, 550,1816,1545,2620,1335, // 2640 -1008, 371,2881, 936,1419,1613,3529,1456,1395,2273,1834,2604,1317,2738,2503, 416, // 2656 -1643,4330, 806,1126, 229, 591,3946,1314,1981,1576,1837,1666, 347,1790, 977,3313, // 2672 - 764,2861,1853, 688,2429,1920,1462, 77, 595, 415,2002,3034, 798,1192,4115,6144, // 2688 -2978,4331,3035,2695,2582,2072,2566, 430,2430,1727, 842,1396,3947,3702, 613, 377, // 2704 - 278, 236,1417,3388,3314,3174, 757,1869, 107,3530,6145,1194, 623,2262, 207,1253, // 2720 -2167,3446,3948, 492,1117,1935, 536,1838,2757,1246,4332, 696,2095,2406,1393,1572, // 2736 -3175,1782, 583, 190, 253,1390,2230, 830,3126,3389, 934,3245,1703,1749,2979,1870, // 2752 -2545,1656,2204, 869,2346,4116,3176,1817, 496,1764,4644, 942,1504, 404,1903,1122, // 2768 -1580,3606,2945,1022, 515, 372,1735, 955,2431,3036,6146,2797,1110,2302,2798, 617, // 2784 -6147, 441, 762,1771,3447,3607,3608,1904, 840,3037, 86, 939,1385, 572,1370,2445, // 2800 -1336, 114,3703, 898, 294, 203,3315, 703,1583,2274, 429, 961,4333,1854,1951,3390, // 2816 -2373,3704,4334,1318,1381, 966,1911,2322,1006,1155, 309, 989, 458,2718,1795,1372, // 2832 -1203, 252,1689,1363,3177, 517,1936, 168,1490, 562, 193,3823,1042,4117,1835, 551, // 2848 - 470,4645, 395, 489,3448,1871,1465,2583,2641, 417,1493, 279,1295, 511,1236,1119, // 2864 - 72,1231,1982,1812,3004, 871,1564, 984,3449,1667,2696,2096,4646,2347,2833,1673, // 2880 -3609, 695,3246,2668, 807,1183,4647, 890, 388,2333,1801,1457,2911,1765,1477,1031, // 2896 -3316,3317,1278,3391,2799,2292,2526, 163,3450,4335,2669,1404,1802,6148,2323,2407, // 2912 -1584,1728,1494,1824,1269, 298, 909,3318,1034,1632, 375, 776,1683,2061, 291, 210, // 2928 -1123, 809,1249,1002,2642,3038, 206,1011,2132, 144, 975, 882,1565, 342, 667, 754, // 2944 -1442,2143,1299,2303,2062, 447, 626,2205,1221,2739,2912,1144,1214,2206,2584, 760, // 2960 -1715, 614, 950,1281,2670,2621, 810, 577,1287,2546,4648, 242,2168, 250,2643, 691, // 2976 - 123,2644, 647, 313,1029, 689,1357,2946,1650, 216, 771,1339,1306, 808,2063, 549, // 2992 - 913,1371,2913,2914,6149,1466,1092,1174,1196,1311,2605,2396,1783,1796,3079, 406, // 3008 -2671,2117,3949,4649, 487,1825,2220,6150,2915, 448,2348,1073,6151,2397,1707, 130, // 3024 - 900,1598, 329, 176,1959,2527,1620,6152,2275,4336,3319,1983,2191,3705,3610,2155, // 3040 -3706,1912,1513,1614,6153,1988, 646, 392,2304,1589,3320,3039,1826,1239,1352,1340, // 3056 -2916, 505,2567,1709,1437,2408,2547, 906,6154,2672, 384,1458,1594,1100,1329, 710, // 3072 - 423,3531,2064,2231,2622,1989,2673,1087,1882, 333, 841,3005,1296,2882,2379, 580, // 3088 -1937,1827,1293,2585, 601, 574, 249,1772,4118,2079,1120, 645, 901,1176,1690, 795, // 3104 -2207, 478,1434, 516,1190,1530, 761,2080, 930,1264, 355, 435,1552, 644,1791, 987, // 3120 - 220,1364,1163,1121,1538, 306,2169,1327,1222, 546,2645, 218, 241, 610,1704,3321, // 3136 -1984,1839,1966,2528, 451,6155,2586,3707,2568, 907,3178, 254,2947, 186,1845,4650, // 3152 - 745, 432,1757, 428,1633, 888,2246,2221,2489,3611,2118,1258,1265, 956,3127,1784, // 3168 -4337,2490, 319, 510, 119, 457,3612, 274,2035,2007,4651,1409,3128, 970,2758, 590, // 3184 -2800, 661,2247,4652,2008,3950,1420,1549,3080,3322,3951,1651,1375,2111, 485,2491, // 3200 -1429,1156,6156,2548,2183,1495, 831,1840,2529,2446, 501,1657, 307,1894,3247,1341, // 3216 - 666, 899,2156,1539,2549,1559, 886, 349,2208,3081,2305,1736,3824,2170,2759,1014, // 3232 -1913,1386, 542,1397,2948, 490, 368, 716, 362, 159, 282,2569,1129,1658,1288,1750, // 3248 -2674, 276, 649,2016, 751,1496, 658,1818,1284,1862,2209,2087,2512,3451, 622,2834, // 3264 - 376, 117,1060,2053,1208,1721,1101,1443, 247,1250,3179,1792,3952,2760,2398,3953, // 3280 -6157,2144,3708, 446,2432,1151,2570,3452,2447,2761,2835,1210,2448,3082, 424,2222, // 3296 -1251,2449,2119,2836, 504,1581,4338, 602, 817, 857,3825,2349,2306, 357,3826,1470, // 3312 -1883,2883, 255, 958, 929,2917,3248, 302,4653,1050,1271,1751,2307,1952,1430,2697, // 3328 -2719,2359, 354,3180, 777, 158,2036,4339,1659,4340,4654,2308,2949,2248,1146,2232, // 3344 -3532,2720,1696,2623,3827,6158,3129,1550,2698,1485,1297,1428, 637, 931,2721,2145, // 3360 - 914,2550,2587, 81,2450, 612, 827,2646,1242,4655,1118,2884, 472,1855,3181,3533, // 3376 -3534, 569,1353,2699,1244,1758,2588,4119,2009,2762,2171,3709,1312,1531,6159,1152, // 3392 -1938, 134,1830, 471,3710,2276,1112,1535,3323,3453,3535, 982,1337,2950, 488, 826, // 3408 - 674,1058,1628,4120,2017, 522,2399, 211, 568,1367,3454, 350, 293,1872,1139,3249, // 3424 -1399,1946,3006,1300,2360,3324, 588, 736,6160,2606, 744, 669,3536,3828,6161,1358, // 3440 - 199, 723, 848, 933, 851,1939,1505,1514,1338,1618,1831,4656,1634,3613, 443,2740, // 3456 -3829, 717,1947, 491,1914,6162,2551,1542,4121,1025,6163,1099,1223, 198,3040,2722, // 3472 - 370, 410,1905,2589, 998,1248,3182,2380, 519,1449,4122,1710, 947, 928,1153,4341, // 3488 -2277, 344,2624,1511, 615, 105, 161,1212,1076,1960,3130,2054,1926,1175,1906,2473, // 3504 - 414,1873,2801,6164,2309, 315,1319,3325, 318,2018,2146,2157, 963, 631, 223,4342, // 3520 -4343,2675, 479,3711,1197,2625,3712,2676,2361,6165,4344,4123,6166,2451,3183,1886, // 3536 -2184,1674,1330,1711,1635,1506, 799, 219,3250,3083,3954,1677,3713,3326,2081,3614, // 3552 -1652,2073,4657,1147,3041,1752, 643,1961, 147,1974,3955,6167,1716,2037, 918,3007, // 3568 -1994, 120,1537, 118, 609,3184,4345, 740,3455,1219, 332,1615,3830,6168,1621,2980, // 3584 -1582, 783, 212, 553,2350,3714,1349,2433,2082,4124, 889,6169,2310,1275,1410, 973, // 3600 - 166,1320,3456,1797,1215,3185,2885,1846,2590,2763,4658, 629, 822,3008, 763, 940, // 3616 -1990,2862, 439,2409,1566,1240,1622, 926,1282,1907,2764, 654,2210,1607, 327,1130, // 3632 -3956,1678,1623,6170,2434,2192, 686, 608,3831,3715, 903,3957,3042,6171,2741,1522, // 3648 -1915,1105,1555,2552,1359, 323,3251,4346,3457, 738,1354,2553,2311,2334,1828,2003, // 3664 -3832,1753,2351,1227,6172,1887,4125,1478,6173,2410,1874,1712,1847, 520,1204,2607, // 3680 - 264,4659, 836,2677,2102, 600,4660,3833,2278,3084,6174,4347,3615,1342, 640, 532, // 3696 - 543,2608,1888,2400,2591,1009,4348,1497, 341,1737,3616,2723,1394, 529,3252,1321, // 3712 - 983,4661,1515,2120, 971,2592, 924, 287,1662,3186,4349,2700,4350,1519, 908,1948, // 3728 -2452, 156, 796,1629,1486,2223,2055, 694,4126,1259,1036,3392,1213,2249,2742,1889, // 3744 -1230,3958,1015, 910, 408, 559,3617,4662, 746, 725, 935,4663,3959,3009,1289, 563, // 3760 - 867,4664,3960,1567,2981,2038,2626, 988,2263,2381,4351, 143,2374, 704,1895,6175, // 3776 -1188,3716,2088, 673,3085,2362,4352, 484,1608,1921,2765,2918, 215, 904,3618,3537, // 3792 - 894, 509, 976,3043,2701,3961,4353,2837,2982, 498,6176,6177,1102,3538,1332,3393, // 3808 -1487,1636,1637, 233, 245,3962, 383, 650, 995,3044, 460,1520,1206,2352, 749,3327, // 3824 - 530, 700, 389,1438,1560,1773,3963,2264, 719,2951,2724,3834, 870,1832,1644,1000, // 3840 - 839,2474,3717, 197,1630,3394, 365,2886,3964,1285,2133, 734, 922, 818,1106, 732, // 3856 - 480,2083,1774,3458, 923,2279,1350, 221,3086, 85,2233,2234,3835,1585,3010,2147, // 3872 -1387,1705,2382,1619,2475, 133, 239,2802,1991,1016,2084,2383, 411,2838,1113, 651, // 3888 -1985,1160,3328, 990,1863,3087,1048,1276,2647, 265,2627,1599,3253,2056, 150, 638, // 3904 -2019, 656, 853, 326,1479, 680,1439,4354,1001,1759, 413,3459,3395,2492,1431, 459, // 3920 -4355,1125,3329,2265,1953,1450,2065,2863, 849, 351,2678,3131,3254,3255,1104,1577, // 3936 - 227,1351,1645,2453,2193,1421,2887, 812,2121, 634, 95,2435, 201,2312,4665,1646, // 3952 -1671,2743,1601,2554,2702,2648,2280,1315,1366,2089,3132,1573,3718,3965,1729,1189, // 3968 - 328,2679,1077,1940,1136, 558,1283, 964,1195, 621,2074,1199,1743,3460,3619,1896, // 3984 -1916,1890,3836,2952,1154,2112,1064, 862, 378,3011,2066,2113,2803,1568,2839,6178, // 4000 -3088,2919,1941,1660,2004,1992,2194, 142, 707,1590,1708,1624,1922,1023,1836,1233, // 4016 -1004,2313, 789, 741,3620,6179,1609,2411,1200,4127,3719,3720,4666,2057,3721, 593, // 4032 -2840, 367,2920,1878,6180,3461,1521, 628,1168, 692,2211,2649, 300, 720,2067,2571, // 4048 -2953,3396, 959,2504,3966,3539,3462,1977, 701,6181, 954,1043, 800, 681, 183,3722, // 4064 -1803,1730,3540,4128,2103, 815,2314, 174, 467, 230,2454,1093,2134, 755,3541,3397, // 4080 -1141,1162,6182,1738,2039, 270,3256,2513,1005,1647,2185,3837, 858,1679,1897,1719, // 4096 -2954,2324,1806, 402, 670, 167,4129,1498,2158,2104, 750,6183, 915, 189,1680,1551, // 4112 - 455,4356,1501,2455, 405,1095,2955, 338,1586,1266,1819, 570, 641,1324, 237,1556, // 4128 -2650,1388,3723,6184,1368,2384,1343,1978,3089,2436, 879,3724, 792,1191, 758,3012, // 4144 -1411,2135,1322,4357, 240,4667,1848,3725,1574,6185, 420,3045,1546,1391, 714,4358, // 4160 -1967, 941,1864, 863, 664, 426, 560,1731,2680,1785,2864,1949,2363, 403,3330,1415, // 4176 -1279,2136,1697,2335, 204, 721,2097,3838, 90,6186,2085,2505, 191,3967, 124,2148, // 4192 -1376,1798,1178,1107,1898,1405, 860,4359,1243,1272,2375,2983,1558,2456,1638, 113, // 4208 -3621, 578,1923,2609, 880, 386,4130, 784,2186,2266,1422,2956,2172,1722, 497, 263, // 4224 -2514,1267,2412,2610, 177,2703,3542, 774,1927,1344, 616,1432,1595,1018, 172,4360, // 4240 -2325, 911,4361, 438,1468,3622, 794,3968,2024,2173,1681,1829,2957, 945, 895,3090, // 4256 - 575,2212,2476, 475,2401,2681, 785,2744,1745,2293,2555,1975,3133,2865, 394,4668, // 4272 -3839, 635,4131, 639, 202,1507,2195,2766,1345,1435,2572,3726,1908,1184,1181,2457, // 4288 -3727,3134,4362, 843,2611, 437, 916,4669, 234, 769,1884,3046,3047,3623, 833,6187, // 4304 -1639,2250,2402,1355,1185,2010,2047, 999, 525,1732,1290,1488,2612, 948,1578,3728, // 4320 -2413,2477,1216,2725,2159, 334,3840,1328,3624,2921,1525,4132, 564,1056, 891,4363, // 4336 -1444,1698,2385,2251,3729,1365,2281,2235,1717,6188, 864,3841,2515, 444, 527,2767, // 4352 -2922,3625, 544, 461,6189, 566, 209,2437,3398,2098,1065,2068,3331,3626,3257,2137, // 4368 //last 512 - -/*************************************************************************************** - *Everything below is of no interest for detection purpose * - *************************************************************************************** - -2138,2122,3730,2888,1995,1820,1044,6190,6191,6192,6193,6194,6195,6196,6197,6198, // 4384 -6199,6200,6201,6202,6203,6204,6205,4670,6206,6207,6208,6209,6210,6211,6212,6213, // 4400 -6214,6215,6216,6217,6218,6219,6220,6221,6222,6223,6224,6225,6226,6227,6228,6229, // 4416 -6230,6231,6232,6233,6234,6235,6236,6237,3187,6238,6239,3969,6240,6241,6242,6243, // 4432 -6244,4671,6245,6246,4672,6247,6248,4133,6249,6250,4364,6251,2923,2556,2613,4673, // 4448 -4365,3970,6252,6253,6254,6255,4674,6256,6257,6258,2768,2353,4366,4675,4676,3188, // 4464 -4367,3463,6259,4134,4677,4678,6260,2267,6261,3842,3332,4368,3543,6262,6263,6264, // 4480 -3013,1954,1928,4135,4679,6265,6266,2478,3091,6267,4680,4369,6268,6269,1699,6270, // 4496 -3544,4136,4681,6271,4137,6272,4370,2804,6273,6274,2593,3971,3972,4682,6275,2236, // 4512 -4683,6276,6277,4684,6278,6279,4138,3973,4685,6280,6281,3258,6282,6283,6284,6285, // 4528 -3974,4686,2841,3975,6286,6287,3545,6288,6289,4139,4687,4140,6290,4141,6291,4142, // 4544 -6292,6293,3333,6294,6295,6296,4371,6297,3399,6298,6299,4372,3976,6300,6301,6302, // 4560 -4373,6303,6304,3843,3731,6305,4688,4374,6306,6307,3259,2294,6308,3732,2530,4143, // 4576 -6309,4689,6310,6311,6312,3048,6313,6314,4690,3733,2237,6315,6316,2282,3334,6317, // 4592 -6318,3844,6319,6320,4691,6321,3400,4692,6322,4693,6323,3049,6324,4375,6325,3977, // 4608 -6326,6327,6328,3546,6329,4694,3335,6330,4695,4696,6331,6332,6333,6334,4376,3978, // 4624 -6335,4697,3979,4144,6336,3980,4698,6337,6338,6339,6340,6341,4699,4700,4701,6342, // 4640 -6343,4702,6344,6345,4703,6346,6347,4704,6348,4705,4706,3135,6349,4707,6350,4708, // 4656 -6351,4377,6352,4709,3734,4145,6353,2506,4710,3189,6354,3050,4711,3981,6355,3547, // 4672 -3014,4146,4378,3735,2651,3845,3260,3136,2224,1986,6356,3401,6357,4712,2594,3627, // 4688 -3137,2573,3736,3982,4713,3628,4714,4715,2682,3629,4716,6358,3630,4379,3631,6359, // 4704 -6360,6361,3983,6362,6363,6364,6365,4147,3846,4717,6366,6367,3737,2842,6368,4718, // 4720 -2628,6369,3261,6370,2386,6371,6372,3738,3984,4719,3464,4720,3402,6373,2924,3336, // 4736 -4148,2866,6374,2805,3262,4380,2704,2069,2531,3138,2806,2984,6375,2769,6376,4721, // 4752 -4722,3403,6377,6378,3548,6379,6380,2705,3092,1979,4149,2629,3337,2889,6381,3338, // 4768 -4150,2557,3339,4381,6382,3190,3263,3739,6383,4151,4723,4152,2558,2574,3404,3191, // 4784 -6384,6385,4153,6386,4724,4382,6387,6388,4383,6389,6390,4154,6391,4725,3985,6392, // 4800 -3847,4155,6393,6394,6395,6396,6397,3465,6398,4384,6399,6400,6401,6402,6403,6404, // 4816 -4156,6405,6406,6407,6408,2123,6409,6410,2326,3192,4726,6411,6412,6413,6414,4385, // 4832 -4157,6415,6416,4158,6417,3093,3848,6418,3986,6419,6420,3849,6421,6422,6423,4159, // 4848 -6424,6425,4160,6426,3740,6427,6428,6429,6430,3987,6431,4727,6432,2238,6433,6434, // 4864 -4386,3988,6435,6436,3632,6437,6438,2843,6439,6440,6441,6442,3633,6443,2958,6444, // 4880 -6445,3466,6446,2364,4387,3850,6447,4388,2959,3340,6448,3851,6449,4728,6450,6451, // 4896 -3264,4729,6452,3193,6453,4389,4390,2706,3341,4730,6454,3139,6455,3194,6456,3051, // 4912 -2124,3852,1602,4391,4161,3853,1158,3854,4162,3989,4392,3990,4731,4732,4393,2040, // 4928 -4163,4394,3265,6457,2807,3467,3855,6458,6459,6460,3991,3468,4733,4734,6461,3140, // 4944 -2960,6462,4735,6463,6464,6465,6466,4736,4737,4738,4739,6467,6468,4164,2403,3856, // 4960 -6469,6470,2770,2844,6471,4740,6472,6473,6474,6475,6476,6477,6478,3195,6479,4741, // 4976 -4395,6480,2867,6481,4742,2808,6482,2493,4165,6483,6484,6485,6486,2295,4743,6487, // 4992 -6488,6489,3634,6490,6491,6492,6493,6494,6495,6496,2985,4744,6497,6498,4745,6499, // 5008 -6500,2925,3141,4166,6501,6502,4746,6503,6504,4747,6505,6506,6507,2890,6508,6509, // 5024 -6510,6511,6512,6513,6514,6515,6516,6517,6518,6519,3469,4167,6520,6521,6522,4748, // 5040 -4396,3741,4397,4749,4398,3342,2125,4750,6523,4751,4752,4753,3052,6524,2961,4168, // 5056 -6525,4754,6526,4755,4399,2926,4169,6527,3857,6528,4400,4170,6529,4171,6530,6531, // 5072 -2595,6532,6533,6534,6535,3635,6536,6537,6538,6539,6540,6541,6542,4756,6543,6544, // 5088 -6545,6546,6547,6548,4401,6549,6550,6551,6552,4402,3405,4757,4403,6553,6554,6555, // 5104 -4172,3742,6556,6557,6558,3992,3636,6559,6560,3053,2726,6561,3549,4173,3054,4404, // 5120 -6562,6563,3993,4405,3266,3550,2809,4406,6564,6565,6566,4758,4759,6567,3743,6568, // 5136 -4760,3744,4761,3470,6569,6570,6571,4407,6572,3745,4174,6573,4175,2810,4176,3196, // 5152 -4762,6574,4177,6575,6576,2494,2891,3551,6577,6578,3471,6579,4408,6580,3015,3197, // 5168 -6581,3343,2532,3994,3858,6582,3094,3406,4409,6583,2892,4178,4763,4410,3016,4411, // 5184 -6584,3995,3142,3017,2683,6585,4179,6586,6587,4764,4412,6588,6589,4413,6590,2986, // 5200 -6591,2962,3552,6592,2963,3472,6593,6594,4180,4765,6595,6596,2225,3267,4414,6597, // 5216 -3407,3637,4766,6598,6599,3198,6600,4415,6601,3859,3199,6602,3473,4767,2811,4416, // 5232 -1856,3268,3200,2575,3996,3997,3201,4417,6603,3095,2927,6604,3143,6605,2268,6606, // 5248 -3998,3860,3096,2771,6607,6608,3638,2495,4768,6609,3861,6610,3269,2745,4769,4181, // 5264 -3553,6611,2845,3270,6612,6613,6614,3862,6615,6616,4770,4771,6617,3474,3999,4418, // 5280 -4419,6618,3639,3344,6619,4772,4182,6620,2126,6621,6622,6623,4420,4773,6624,3018, // 5296 -6625,4774,3554,6626,4183,2025,3746,6627,4184,2707,6628,4421,4422,3097,1775,4185, // 5312 -3555,6629,6630,2868,6631,6632,4423,6633,6634,4424,2414,2533,2928,6635,4186,2387, // 5328 -6636,4775,6637,4187,6638,1891,4425,3202,3203,6639,6640,4776,6641,3345,6642,6643, // 5344 -3640,6644,3475,3346,3641,4000,6645,3144,6646,3098,2812,4188,3642,3204,6647,3863, // 5360 -3476,6648,3864,6649,4426,4001,6650,6651,6652,2576,6653,4189,4777,6654,6655,6656, // 5376 -2846,6657,3477,3205,4002,6658,4003,6659,3347,2252,6660,6661,6662,4778,6663,6664, // 5392 -6665,6666,6667,6668,6669,4779,4780,2048,6670,3478,3099,6671,3556,3747,4004,6672, // 5408 -6673,6674,3145,4005,3748,6675,6676,6677,6678,6679,3408,6680,6681,6682,6683,3206, // 5424 -3207,6684,6685,4781,4427,6686,4782,4783,4784,6687,6688,6689,4190,6690,6691,3479, // 5440 -6692,2746,6693,4428,6694,6695,6696,6697,6698,6699,4785,6700,6701,3208,2727,6702, // 5456 -3146,6703,6704,3409,2196,6705,4429,6706,6707,6708,2534,1996,6709,6710,6711,2747, // 5472 -6712,6713,6714,4786,3643,6715,4430,4431,6716,3557,6717,4432,4433,6718,6719,6720, // 5488 -6721,3749,6722,4006,4787,6723,6724,3644,4788,4434,6725,6726,4789,2772,6727,6728, // 5504 -6729,6730,6731,2708,3865,2813,4435,6732,6733,4790,4791,3480,6734,6735,6736,6737, // 5520 -4436,3348,6738,3410,4007,6739,6740,4008,6741,6742,4792,3411,4191,6743,6744,6745, // 5536 -6746,6747,3866,6748,3750,6749,6750,6751,6752,6753,6754,6755,3867,6756,4009,6757, // 5552 -4793,4794,6758,2814,2987,6759,6760,6761,4437,6762,6763,6764,6765,3645,6766,6767, // 5568 -3481,4192,6768,3751,6769,6770,2174,6771,3868,3752,6772,6773,6774,4193,4795,4438, // 5584 -3558,4796,4439,6775,4797,6776,6777,4798,6778,4799,3559,4800,6779,6780,6781,3482, // 5600 -6782,2893,6783,6784,4194,4801,4010,6785,6786,4440,6787,4011,6788,6789,6790,6791, // 5616 -6792,6793,4802,6794,6795,6796,4012,6797,6798,6799,6800,3349,4803,3483,6801,4804, // 5632 -4195,6802,4013,6803,6804,4196,6805,4014,4015,6806,2847,3271,2848,6807,3484,6808, // 5648 -6809,6810,4441,6811,4442,4197,4443,3272,4805,6812,3412,4016,1579,6813,6814,4017, // 5664 -6815,3869,6816,2964,6817,4806,6818,6819,4018,3646,6820,6821,4807,4019,4020,6822, // 5680 -6823,3560,6824,6825,4021,4444,6826,4198,6827,6828,4445,6829,6830,4199,4808,6831, // 5696 -6832,6833,3870,3019,2458,6834,3753,3413,3350,6835,4809,3871,4810,3561,4446,6836, // 5712 -6837,4447,4811,4812,6838,2459,4448,6839,4449,6840,6841,4022,3872,6842,4813,4814, // 5728 -6843,6844,4815,4200,4201,4202,6845,4023,6846,6847,4450,3562,3873,6848,6849,4816, // 5744 -4817,6850,4451,4818,2139,6851,3563,6852,6853,3351,6854,6855,3352,4024,2709,3414, // 5760 -4203,4452,6856,4204,6857,6858,3874,3875,6859,6860,4819,6861,6862,6863,6864,4453, // 5776 -3647,6865,6866,4820,6867,6868,6869,6870,4454,6871,2869,6872,6873,4821,6874,3754, // 5792 -6875,4822,4205,6876,6877,6878,3648,4206,4455,6879,4823,6880,4824,3876,6881,3055, // 5808 -4207,6882,3415,6883,6884,6885,4208,4209,6886,4210,3353,6887,3354,3564,3209,3485, // 5824 -2652,6888,2728,6889,3210,3755,6890,4025,4456,6891,4825,6892,6893,6894,6895,4211, // 5840 -6896,6897,6898,4826,6899,6900,4212,6901,4827,6902,2773,3565,6903,4828,6904,6905, // 5856 -6906,6907,3649,3650,6908,2849,3566,6909,3567,3100,6910,6911,6912,6913,6914,6915, // 5872 -4026,6916,3355,4829,3056,4457,3756,6917,3651,6918,4213,3652,2870,6919,4458,6920, // 5888 -2438,6921,6922,3757,2774,4830,6923,3356,4831,4832,6924,4833,4459,3653,2507,6925, // 5904 -4834,2535,6926,6927,3273,4027,3147,6928,3568,6929,6930,6931,4460,6932,3877,4461, // 5920 -2729,3654,6933,6934,6935,6936,2175,4835,2630,4214,4028,4462,4836,4215,6937,3148, // 5936 -4216,4463,4837,4838,4217,6938,6939,2850,4839,6940,4464,6941,6942,6943,4840,6944, // 5952 -4218,3274,4465,6945,6946,2710,6947,4841,4466,6948,6949,2894,6950,6951,4842,6952, // 5968 -4219,3057,2871,6953,6954,6955,6956,4467,6957,2711,6958,6959,6960,3275,3101,4843, // 5984 -6961,3357,3569,6962,4844,6963,6964,4468,4845,3570,6965,3102,4846,3758,6966,4847, // 6000 -3878,4848,4849,4029,6967,2929,3879,4850,4851,6968,6969,1733,6970,4220,6971,6972, // 6016 -6973,6974,6975,6976,4852,6977,6978,6979,6980,6981,6982,3759,6983,6984,6985,3486, // 6032 -3487,6986,3488,3416,6987,6988,6989,6990,6991,6992,6993,6994,6995,6996,6997,4853, // 6048 -6998,6999,4030,7000,7001,3211,7002,7003,4221,7004,7005,3571,4031,7006,3572,7007, // 6064 -2614,4854,2577,7008,7009,2965,3655,3656,4855,2775,3489,3880,4222,4856,3881,4032, // 6080 -3882,3657,2730,3490,4857,7010,3149,7011,4469,4858,2496,3491,4859,2283,7012,7013, // 6096 -7014,2365,4860,4470,7015,7016,3760,7017,7018,4223,1917,7019,7020,7021,4471,7022, // 6112 -2776,4472,7023,7024,7025,7026,4033,7027,3573,4224,4861,4034,4862,7028,7029,1929, // 6128 -3883,4035,7030,4473,3058,7031,2536,3761,3884,7032,4036,7033,2966,2895,1968,4474, // 6144 -3276,4225,3417,3492,4226,2105,7034,7035,1754,2596,3762,4227,4863,4475,3763,4864, // 6160 -3764,2615,2777,3103,3765,3658,3418,4865,2296,3766,2815,7036,7037,7038,3574,2872, // 6176 -3277,4476,7039,4037,4477,7040,7041,4038,7042,7043,7044,7045,7046,7047,2537,7048, // 6192 -7049,7050,7051,7052,7053,7054,4478,7055,7056,3767,3659,4228,3575,7057,7058,4229, // 6208 -7059,7060,7061,3660,7062,3212,7063,3885,4039,2460,7064,7065,7066,7067,7068,7069, // 6224 -7070,7071,7072,7073,7074,4866,3768,4867,7075,7076,7077,7078,4868,3358,3278,2653, // 6240 -7079,7080,4479,3886,7081,7082,4869,7083,7084,7085,7086,7087,7088,2538,7089,7090, // 6256 -7091,4040,3150,3769,4870,4041,2896,3359,4230,2930,7092,3279,7093,2967,4480,3213, // 6272 -4481,3661,7094,7095,7096,7097,7098,7099,7100,7101,7102,2461,3770,7103,7104,4231, // 6288 -3151,7105,7106,7107,4042,3662,7108,7109,4871,3663,4872,4043,3059,7110,7111,7112, // 6304 -3493,2988,7113,4873,7114,7115,7116,3771,4874,7117,7118,4232,4875,7119,3576,2336, // 6320 -4876,7120,4233,3419,4044,4877,4878,4482,4483,4879,4484,4234,7121,3772,4880,1045, // 6336 -3280,3664,4881,4882,7122,7123,7124,7125,4883,7126,2778,7127,4485,4486,7128,4884, // 6352 -3214,3887,7129,7130,3215,7131,4885,4045,7132,7133,4046,7134,7135,7136,7137,7138, // 6368 -7139,7140,7141,7142,7143,4235,7144,4886,7145,7146,7147,4887,7148,7149,7150,4487, // 6384 -4047,4488,7151,7152,4888,4048,2989,3888,7153,3665,7154,4049,7155,7156,7157,7158, // 6400 -7159,7160,2931,4889,4890,4489,7161,2631,3889,4236,2779,7162,7163,4891,7164,3060, // 6416 -7165,1672,4892,7166,4893,4237,3281,4894,7167,7168,3666,7169,3494,7170,7171,4050, // 6432 -7172,7173,3104,3360,3420,4490,4051,2684,4052,7174,4053,7175,7176,7177,2253,4054, // 6448 -7178,7179,4895,7180,3152,3890,3153,4491,3216,7181,7182,7183,2968,4238,4492,4055, // 6464 -7184,2990,7185,2479,7186,7187,4493,7188,7189,7190,7191,7192,4896,7193,4897,2969, // 6480 -4494,4898,7194,3495,7195,7196,4899,4495,7197,3105,2731,7198,4900,7199,7200,7201, // 6496 -4056,7202,3361,7203,7204,4496,4901,4902,7205,4497,7206,7207,2315,4903,7208,4904, // 6512 -7209,4905,2851,7210,7211,3577,7212,3578,4906,7213,4057,3667,4907,7214,4058,2354, // 6528 -3891,2376,3217,3773,7215,7216,7217,7218,7219,4498,7220,4908,3282,2685,7221,3496, // 6544 -4909,2632,3154,4910,7222,2337,7223,4911,7224,7225,7226,4912,4913,3283,4239,4499, // 6560 -7227,2816,7228,7229,7230,7231,7232,7233,7234,4914,4500,4501,7235,7236,7237,2686, // 6576 -7238,4915,7239,2897,4502,7240,4503,7241,2516,7242,4504,3362,3218,7243,7244,7245, // 6592 -4916,7246,7247,4505,3363,7248,7249,7250,7251,3774,4506,7252,7253,4917,7254,7255, // 6608 -3284,2991,4918,4919,3219,3892,4920,3106,3497,4921,7256,7257,7258,4922,7259,4923, // 6624 -3364,4507,4508,4059,7260,4240,3498,7261,7262,4924,7263,2992,3893,4060,3220,7264, // 6640 -7265,7266,7267,7268,7269,4509,3775,7270,2817,7271,4061,4925,4510,3776,7272,4241, // 6656 -4511,3285,7273,7274,3499,7275,7276,7277,4062,4512,4926,7278,3107,3894,7279,7280, // 6672 -4927,7281,4513,7282,7283,3668,7284,7285,4242,4514,4243,7286,2058,4515,4928,4929, // 6688 -4516,7287,3286,4244,7288,4517,7289,7290,7291,3669,7292,7293,4930,4931,4932,2355, // 6704 -4933,7294,2633,4518,7295,4245,7296,7297,4519,7298,7299,4520,4521,4934,7300,4246, // 6720 -4522,7301,7302,7303,3579,7304,4247,4935,7305,4936,7306,7307,7308,7309,3777,7310, // 6736 -4523,7311,7312,7313,4248,3580,7314,4524,3778,4249,7315,3581,7316,3287,7317,3221, // 6752 -7318,4937,7319,7320,7321,7322,7323,7324,4938,4939,7325,4525,7326,7327,7328,4063, // 6768 -7329,7330,4940,7331,7332,4941,7333,4526,7334,3500,2780,1741,4942,2026,1742,7335, // 6784 -7336,3582,4527,2388,7337,7338,7339,4528,7340,4250,4943,7341,7342,7343,4944,7344, // 6800 -7345,7346,3020,7347,4945,7348,7349,7350,7351,3895,7352,3896,4064,3897,7353,7354, // 6816 -7355,4251,7356,7357,3898,7358,3779,7359,3780,3288,7360,7361,4529,7362,4946,4530, // 6832 -2027,7363,3899,4531,4947,3222,3583,7364,4948,7365,7366,7367,7368,4949,3501,4950, // 6848 -3781,4951,4532,7369,2517,4952,4252,4953,3155,7370,4954,4955,4253,2518,4533,7371, // 6864 -7372,2712,4254,7373,7374,7375,3670,4956,3671,7376,2389,3502,4065,7377,2338,7378, // 6880 -7379,7380,7381,3061,7382,4957,7383,7384,7385,7386,4958,4534,7387,7388,2993,7389, // 6896 -3062,7390,4959,7391,7392,7393,4960,3108,4961,7394,4535,7395,4962,3421,4536,7396, // 6912 -4963,7397,4964,1857,7398,4965,7399,7400,2176,3584,4966,7401,7402,3422,4537,3900, // 6928 -3585,7403,3782,7404,2852,7405,7406,7407,4538,3783,2654,3423,4967,4539,7408,3784, // 6944 -3586,2853,4540,4541,7409,3901,7410,3902,7411,7412,3785,3109,2327,3903,7413,7414, // 6960 -2970,4066,2932,7415,7416,7417,3904,3672,3424,7418,4542,4543,4544,7419,4968,7420, // 6976 -7421,4255,7422,7423,7424,7425,7426,4067,7427,3673,3365,4545,7428,3110,2559,3674, // 6992 -7429,7430,3156,7431,7432,3503,7433,3425,4546,7434,3063,2873,7435,3223,4969,4547, // 7008 -4548,2898,4256,4068,7436,4069,3587,3786,2933,3787,4257,4970,4971,3788,7437,4972, // 7024 -3064,7438,4549,7439,7440,7441,7442,7443,4973,3905,7444,2874,7445,7446,7447,7448, // 7040 -3021,7449,4550,3906,3588,4974,7450,7451,3789,3675,7452,2578,7453,4070,7454,7455, // 7056 -7456,4258,3676,7457,4975,7458,4976,4259,3790,3504,2634,4977,3677,4551,4260,7459, // 7072 -7460,7461,7462,3907,4261,4978,7463,7464,7465,7466,4979,4980,7467,7468,2213,4262, // 7088 -7469,7470,7471,3678,4981,7472,2439,7473,4263,3224,3289,7474,3908,2415,4982,7475, // 7104 -4264,7476,4983,2655,7477,7478,2732,4552,2854,2875,7479,7480,4265,7481,4553,4984, // 7120 -7482,7483,4266,7484,3679,3366,3680,2818,2781,2782,3367,3589,4554,3065,7485,4071, // 7136 -2899,7486,7487,3157,2462,4072,4555,4073,4985,4986,3111,4267,2687,3368,4556,4074, // 7152 -3791,4268,7488,3909,2783,7489,2656,1962,3158,4557,4987,1963,3159,3160,7490,3112, // 7168 -4988,4989,3022,4990,4991,3792,2855,7491,7492,2971,4558,7493,7494,4992,7495,7496, // 7184 -7497,7498,4993,7499,3426,4559,4994,7500,3681,4560,4269,4270,3910,7501,4075,4995, // 7200 -4271,7502,7503,4076,7504,4996,7505,3225,4997,4272,4077,2819,3023,7506,7507,2733, // 7216 -4561,7508,4562,7509,3369,3793,7510,3590,2508,7511,7512,4273,3113,2994,2616,7513, // 7232 -7514,7515,7516,7517,7518,2820,3911,4078,2748,7519,7520,4563,4998,7521,7522,7523, // 7248 -7524,4999,4274,7525,4564,3682,2239,4079,4565,7526,7527,7528,7529,5000,7530,7531, // 7264 -5001,4275,3794,7532,7533,7534,3066,5002,4566,3161,7535,7536,4080,7537,3162,7538, // 7280 -7539,4567,7540,7541,7542,7543,7544,7545,5003,7546,4568,7547,7548,7549,7550,7551, // 7296 -7552,7553,7554,7555,7556,5004,7557,7558,7559,5005,7560,3795,7561,4569,7562,7563, // 7312 -7564,2821,3796,4276,4277,4081,7565,2876,7566,5006,7567,7568,2900,7569,3797,3912, // 7328 -7570,7571,7572,4278,7573,7574,7575,5007,7576,7577,5008,7578,7579,4279,2934,7580, // 7344 -7581,5009,7582,4570,7583,4280,7584,7585,7586,4571,4572,3913,7587,4573,3505,7588, // 7360 -5010,7589,7590,7591,7592,3798,4574,7593,7594,5011,7595,4281,7596,7597,7598,4282, // 7376 -5012,7599,7600,5013,3163,7601,5014,7602,3914,7603,7604,2734,4575,4576,4577,7605, // 7392 -7606,7607,7608,7609,3506,5015,4578,7610,4082,7611,2822,2901,2579,3683,3024,4579, // 7408 -3507,7612,4580,7613,3226,3799,5016,7614,7615,7616,7617,7618,7619,7620,2995,3290, // 7424 -7621,4083,7622,5017,7623,7624,7625,7626,7627,4581,3915,7628,3291,7629,5018,7630, // 7440 -7631,7632,7633,4084,7634,7635,3427,3800,7636,7637,4582,7638,5019,4583,5020,7639, // 7456 -3916,7640,3801,5021,4584,4283,7641,7642,3428,3591,2269,7643,2617,7644,4585,3592, // 7472 -7645,4586,2902,7646,7647,3227,5022,7648,4587,7649,4284,7650,7651,7652,4588,2284, // 7488 -7653,5023,7654,7655,7656,4589,5024,3802,7657,7658,5025,3508,4590,7659,7660,7661, // 7504 -1969,5026,7662,7663,3684,1821,2688,7664,2028,2509,4285,7665,2823,1841,7666,2689, // 7520 -3114,7667,3917,4085,2160,5027,5028,2972,7668,5029,7669,7670,7671,3593,4086,7672, // 7536 -4591,4087,5030,3803,7673,7674,7675,7676,7677,7678,7679,4286,2366,4592,4593,3067, // 7552 -2328,7680,7681,4594,3594,3918,2029,4287,7682,5031,3919,3370,4288,4595,2856,7683, // 7568 -3509,7684,7685,5032,5033,7686,7687,3804,2784,7688,7689,7690,7691,3371,7692,7693, // 7584 -2877,5034,7694,7695,3920,4289,4088,7696,7697,7698,5035,7699,5036,4290,5037,5038, // 7600 -5039,7700,7701,7702,5040,5041,3228,7703,1760,7704,5042,3229,4596,2106,4089,7705, // 7616 -4597,2824,5043,2107,3372,7706,4291,4090,5044,7707,4091,7708,5045,3025,3805,4598, // 7632 -4292,4293,4294,3373,7709,4599,7710,5046,7711,7712,5047,5048,3806,7713,7714,7715, // 7648 -5049,7716,7717,7718,7719,4600,5050,7720,7721,7722,5051,7723,4295,3429,7724,7725, // 7664 -7726,7727,3921,7728,3292,5052,4092,7729,7730,7731,7732,7733,7734,7735,5053,5054, // 7680 -7736,7737,7738,7739,3922,3685,7740,7741,7742,7743,2635,5055,7744,5056,4601,7745, // 7696 -7746,2560,7747,7748,7749,7750,3923,7751,7752,7753,7754,7755,4296,2903,7756,7757, // 7712 -7758,7759,7760,3924,7761,5057,4297,7762,7763,5058,4298,7764,4093,7765,7766,5059, // 7728 -3925,7767,7768,7769,7770,7771,7772,7773,7774,7775,7776,3595,7777,4299,5060,4094, // 7744 -7778,3293,5061,7779,7780,4300,7781,7782,4602,7783,3596,7784,7785,3430,2367,7786, // 7760 -3164,5062,5063,4301,7787,7788,4095,5064,5065,7789,3374,3115,7790,7791,7792,7793, // 7776 -7794,7795,7796,3597,4603,7797,7798,3686,3116,3807,5066,7799,7800,5067,7801,7802, // 7792 -4604,4302,5068,4303,4096,7803,7804,3294,7805,7806,5069,4605,2690,7807,3026,7808, // 7808 -7809,7810,7811,7812,7813,7814,7815,7816,7817,7818,7819,7820,7821,7822,7823,7824, // 7824 -7825,7826,7827,7828,7829,7830,7831,7832,7833,7834,7835,7836,7837,7838,7839,7840, // 7840 -7841,7842,7843,7844,7845,7846,7847,7848,7849,7850,7851,7852,7853,7854,7855,7856, // 7856 -7857,7858,7859,7860,7861,7862,7863,7864,7865,7866,7867,7868,7869,7870,7871,7872, // 7872 -7873,7874,7875,7876,7877,7878,7879,7880,7881,7882,7883,7884,7885,7886,7887,7888, // 7888 -7889,7890,7891,7892,7893,7894,7895,7896,7897,7898,7899,7900,7901,7902,7903,7904, // 7904 -7905,7906,7907,7908,7909,7910,7911,7912,7913,7914,7915,7916,7917,7918,7919,7920, // 7920 -7921,7922,7923,7924,3926,7925,7926,7927,7928,7929,7930,7931,7932,7933,7934,7935, // 7936 -7936,7937,7938,7939,7940,7941,7942,7943,7944,7945,7946,7947,7948,7949,7950,7951, // 7952 -7952,7953,7954,7955,7956,7957,7958,7959,7960,7961,7962,7963,7964,7965,7966,7967, // 7968 -7968,7969,7970,7971,7972,7973,7974,7975,7976,7977,7978,7979,7980,7981,7982,7983, // 7984 -7984,7985,7986,7987,7988,7989,7990,7991,7992,7993,7994,7995,7996,7997,7998,7999, // 8000 -8000,8001,8002,8003,8004,8005,8006,8007,8008,8009,8010,8011,8012,8013,8014,8015, // 8016 -8016,8017,8018,8019,8020,8021,8022,8023,8024,8025,8026,8027,8028,8029,8030,8031, // 8032 -8032,8033,8034,8035,8036,8037,8038,8039,8040,8041,8042,8043,8044,8045,8046,8047, // 8048 -8048,8049,8050,8051,8052,8053,8054,8055,8056,8057,8058,8059,8060,8061,8062,8063, // 8064 -8064,8065,8066,8067,8068,8069,8070,8071,8072,8073,8074,8075,8076,8077,8078,8079, // 8080 -8080,8081,8082,8083,8084,8085,8086,8087,8088,8089,8090,8091,8092,8093,8094,8095, // 8096 -8096,8097,8098,8099,8100,8101,8102,8103,8104,8105,8106,8107,8108,8109,8110,8111, // 8112 -8112,8113,8114,8115,8116,8117,8118,8119,8120,8121,8122,8123,8124,8125,8126,8127, // 8128 -8128,8129,8130,8131,8132,8133,8134,8135,8136,8137,8138,8139,8140,8141,8142,8143, // 8144 -8144,8145,8146,8147,8148,8149,8150,8151,8152,8153,8154,8155,8156,8157,8158,8159, // 8160 -8160,8161,8162,8163,8164,8165,8166,8167,8168,8169,8170,8171,8172,8173,8174,8175, // 8176 -8176,8177,8178,8179,8180,8181,8182,8183,8184,8185,8186,8187,8188,8189,8190,8191, // 8192 -8192,8193,8194,8195,8196,8197,8198,8199,8200,8201,8202,8203,8204,8205,8206,8207, // 8208 -8208,8209,8210,8211,8212,8213,8214,8215,8216,8217,8218,8219,8220,8221,8222,8223, // 8224 -8224,8225,8226,8227,8228,8229,8230,8231,8232,8233,8234,8235,8236,8237,8238,8239, // 8240 -8240,8241,8242,8243,8244,8245,8246,8247,8248,8249,8250,8251,8252,8253,8254,8255, // 8256 -8256,8257,8258,8259,8260,8261,8262,8263,8264,8265,8266,8267,8268,8269,8270,8271, // 8272 -****************************************************************************************/ - -}; - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/JpCntx.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/JpCntx.cpp deleted file mode 100644 index ecf8087..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/JpCntx.cpp +++ /dev/null @@ -1,230 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nscore.h" -#include "JpCntx.h" - -//This is hiragana 2-char sequence table, the number in each cell represents its frequency category -const PRUint8 jp2CharContext[83][83] = -{ -{ 0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,}, -{ 2,4,0,4,0,3,0,4,0,3,4,4,4,2,4,3,3,4,3,2,3,3,4,2,3,3,3,2,4,1,4,3,3,1,5,4,3,4,3,4,3,5,3,0,3,5,4,2,0,3,1,0,3,3,0,3,3,0,1,1,0,4,3,0,3,3,0,4,0,2,0,3,5,5,5,5,4,0,4,1,0,3,4,}, -{ 0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,}, -{ 0,4,0,5,0,5,0,4,0,4,5,4,4,3,5,3,5,1,5,3,4,3,4,4,3,4,3,3,4,3,5,4,4,3,5,5,3,5,5,5,3,5,5,3,4,5,5,3,1,3,2,0,3,4,0,4,2,0,4,2,1,5,3,2,3,5,0,4,0,2,0,5,4,4,5,4,5,0,4,0,0,4,4,}, -{ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}, -{ 0,3,0,4,0,3,0,3,0,4,5,4,3,3,3,3,4,3,5,4,4,3,5,4,4,3,4,3,4,4,4,4,5,3,4,4,3,4,5,5,4,5,5,1,4,5,4,3,0,3,3,1,3,3,0,4,4,0,3,3,1,5,3,3,3,5,0,4,0,3,0,4,4,3,4,3,3,0,4,1,1,3,4,}, -{ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}, -{ 0,4,0,3,0,3,0,4,0,3,4,4,3,2,2,1,2,1,3,1,3,3,3,3,3,4,3,1,3,3,5,3,3,0,4,3,0,5,4,3,3,5,4,4,3,4,4,5,0,1,2,0,1,2,0,2,2,0,1,0,0,5,2,2,1,4,0,3,0,1,0,4,4,3,5,4,3,0,2,1,0,4,3,}, -{ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}, -{ 0,3,0,5,0,4,0,2,1,4,4,2,4,1,4,2,4,2,4,3,3,3,4,3,3,3,3,1,4,2,3,3,3,1,4,4,1,1,1,4,3,3,2,0,2,4,3,2,0,3,3,0,3,1,1,0,0,0,3,3,0,4,2,2,3,4,0,4,0,3,0,4,4,5,3,4,4,0,3,0,0,1,4,}, -{ 1,4,0,4,0,4,0,4,0,3,5,4,4,3,4,3,5,4,3,3,4,3,5,4,4,4,4,3,4,2,4,3,3,1,5,4,3,2,4,5,4,5,5,4,4,5,4,4,0,3,2,2,3,3,0,4,3,1,3,2,1,4,3,3,4,5,0,3,0,2,0,4,5,5,4,5,4,0,4,0,0,5,4,}, -{ 0,5,0,5,0,4,0,3,0,4,4,3,4,3,3,3,4,0,4,4,4,3,4,3,4,3,3,1,4,2,4,3,4,0,5,4,1,4,5,4,4,5,3,2,4,3,4,3,2,4,1,3,3,3,2,3,2,0,4,3,3,4,3,3,3,4,0,4,0,3,0,4,5,4,4,4,3,0,4,1,0,1,3,}, -{ 0,3,1,4,0,3,0,2,0,3,4,4,3,1,4,2,3,3,4,3,4,3,4,3,4,4,3,2,3,1,5,4,4,1,4,4,3,5,4,4,3,5,5,4,3,4,4,3,1,2,3,1,2,2,0,3,2,0,3,1,0,5,3,3,3,4,3,3,3,3,4,4,4,4,5,4,2,0,3,3,2,4,3,}, -{ 0,2,0,3,0,1,0,1,0,0,3,2,0,0,2,0,1,0,2,1,3,3,3,1,2,3,1,0,1,0,4,2,1,1,3,3,0,4,3,3,1,4,3,3,0,3,3,2,0,0,0,0,1,0,0,2,0,0,0,0,0,4,1,0,2,3,2,2,2,1,3,3,3,4,4,3,2,0,3,1,0,3,3,}, -{ 0,4,0,4,0,3,0,3,0,4,4,4,3,3,3,3,3,3,4,3,4,2,4,3,4,3,3,2,4,3,4,5,4,1,4,5,3,5,4,5,3,5,4,0,3,5,5,3,1,3,3,2,2,3,0,3,4,1,3,3,2,4,3,3,3,4,0,4,0,3,0,4,5,4,4,5,3,0,4,1,0,3,4,}, -{ 0,2,0,3,0,3,0,0,0,2,2,2,1,0,1,0,0,0,3,0,3,0,3,0,1,3,1,0,3,1,3,3,3,1,3,3,3,0,1,3,1,3,4,0,0,3,1,1,0,3,2,0,0,0,0,1,3,0,1,0,0,3,3,2,0,3,0,0,0,0,0,3,4,3,4,3,3,0,3,0,0,2,3,}, -{ 2,3,0,3,0,2,0,1,0,3,3,4,3,1,3,1,1,1,3,1,4,3,4,3,3,3,0,0,3,1,5,4,3,1,4,3,2,5,5,4,4,4,4,3,3,4,4,4,0,2,1,1,3,2,0,1,2,0,0,1,0,4,1,3,3,3,0,3,0,1,0,4,4,4,5,5,3,0,2,0,0,4,4,}, -{ 0,2,0,1,0,3,1,3,0,2,3,3,3,0,3,1,0,0,3,0,3,2,3,1,3,2,1,1,0,0,4,2,1,0,2,3,1,4,3,2,0,4,4,3,1,3,1,3,0,1,0,0,1,0,0,0,1,0,0,0,0,4,1,1,1,2,0,3,0,0,0,3,4,2,4,3,2,0,1,0,0,3,3,}, -{ 0,1,0,4,0,5,0,4,0,2,4,4,2,3,3,2,3,3,5,3,3,3,4,3,4,2,3,0,4,3,3,3,4,1,4,3,2,1,5,5,3,4,5,1,3,5,4,2,0,3,3,0,1,3,0,4,2,0,1,3,1,4,3,3,3,3,0,3,0,1,0,3,4,4,4,5,5,0,3,0,1,4,5,}, -{ 0,2,0,3,0,3,0,0,0,2,3,1,3,0,4,0,1,1,3,0,3,4,3,2,3,1,0,3,3,2,3,1,3,0,2,3,0,2,1,4,1,2,2,0,0,3,3,0,0,2,0,0,0,1,0,0,0,0,2,2,0,3,2,1,3,3,0,2,0,2,0,0,3,3,1,2,4,0,3,0,2,2,3,}, -{ 2,4,0,5,0,4,0,4,0,2,4,4,4,3,4,3,3,3,1,2,4,3,4,3,4,4,5,0,3,3,3,3,2,0,4,3,1,4,3,4,1,4,4,3,3,4,4,3,1,2,3,0,4,2,0,4,1,0,3,3,0,4,3,3,3,4,0,4,0,2,0,3,5,3,4,5,2,0,3,0,0,4,5,}, -{ 0,3,0,4,0,1,0,1,0,1,3,2,2,1,3,0,3,0,2,0,2,0,3,0,2,0,0,0,1,0,1,1,0,0,3,1,0,0,0,4,0,3,1,0,2,1,3,0,0,0,0,0,0,3,0,0,0,0,0,0,0,4,2,2,3,1,0,3,0,0,0,1,4,4,4,3,0,0,4,0,0,1,4,}, -{ 1,4,1,5,0,3,0,3,0,4,5,4,4,3,5,3,3,4,4,3,4,1,3,3,3,3,2,1,4,1,5,4,3,1,4,4,3,5,4,4,3,5,4,3,3,4,4,4,0,3,3,1,2,3,0,3,1,0,3,3,0,5,4,4,4,4,4,4,3,3,5,4,4,3,3,5,4,0,3,2,0,4,4,}, -{ 0,2,0,3,0,1,0,0,0,1,3,3,3,2,4,1,3,0,3,1,3,0,2,2,1,1,0,0,2,0,4,3,1,0,4,3,0,4,4,4,1,4,3,1,1,3,3,1,0,2,0,0,1,3,0,0,0,0,2,0,0,4,3,2,4,3,5,4,3,3,3,4,3,3,4,3,3,0,2,1,0,3,3,}, -{ 0,2,0,4,0,3,0,2,0,2,5,5,3,4,4,4,4,1,4,3,3,0,4,3,4,3,1,3,3,2,4,3,0,3,4,3,0,3,4,4,2,4,4,0,4,5,3,3,2,2,1,1,1,2,0,1,5,0,3,3,2,4,3,3,3,4,0,3,0,2,0,4,4,3,5,5,0,0,3,0,2,3,3,}, -{ 0,3,0,4,0,3,0,1,0,3,4,3,3,1,3,3,3,0,3,1,3,0,4,3,3,1,1,0,3,0,3,3,0,0,4,4,0,1,5,4,3,3,5,0,3,3,4,3,0,2,0,1,1,1,0,1,3,0,1,2,1,3,3,2,3,3,0,3,0,1,0,1,3,3,4,4,1,0,1,2,2,1,3,}, -{ 0,1,0,4,0,4,0,3,0,1,3,3,3,2,3,1,1,0,3,0,3,3,4,3,2,4,2,0,1,0,4,3,2,0,4,3,0,5,3,3,2,4,4,4,3,3,3,4,0,1,3,0,0,1,0,0,1,0,0,0,0,4,2,3,3,3,0,3,0,0,0,4,4,4,5,3,2,0,3,3,0,3,5,}, -{ 0,2,0,3,0,0,0,3,0,1,3,0,2,0,0,0,1,0,3,1,1,3,3,0,0,3,0,0,3,0,2,3,1,0,3,1,0,3,3,2,0,4,2,2,0,2,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,2,1,2,0,1,0,1,0,0,0,1,3,1,2,0,0,0,1,0,0,1,4,}, -{ 0,3,0,3,0,5,0,1,0,2,4,3,1,3,3,2,1,1,5,2,1,0,5,1,2,0,0,0,3,3,2,2,3,2,4,3,0,0,3,3,1,3,3,0,2,5,3,4,0,3,3,0,1,2,0,2,2,0,3,2,0,2,2,3,3,3,0,2,0,1,0,3,4,4,2,5,4,0,3,0,0,3,5,}, -{ 0,3,0,3,0,3,0,1,0,3,3,3,3,0,3,0,2,0,2,1,1,0,2,0,1,0,0,0,2,1,0,0,1,0,3,2,0,0,3,3,1,2,3,1,0,3,3,0,0,1,0,0,0,0,0,2,0,0,0,0,0,2,3,1,2,3,0,3,0,1,0,3,2,1,0,4,3,0,1,1,0,3,3,}, -{ 0,4,0,5,0,3,0,3,0,4,5,5,4,3,5,3,4,3,5,3,3,2,5,3,4,4,4,3,4,3,4,5,5,3,4,4,3,4,4,5,4,4,4,3,4,5,5,4,2,3,4,2,3,4,0,3,3,1,4,3,2,4,3,3,5,5,0,3,0,3,0,5,5,5,5,4,4,0,4,0,1,4,4,}, -{ 0,4,0,4,0,3,0,3,0,3,5,4,4,2,3,2,5,1,3,2,5,1,4,2,3,2,3,3,4,3,3,3,3,2,5,4,1,3,3,5,3,4,4,0,4,4,3,1,1,3,1,0,2,3,0,2,3,0,3,0,0,4,3,1,3,4,0,3,0,2,0,4,4,4,3,4,5,0,4,0,0,3,4,}, -{ 0,3,0,3,0,3,1,2,0,3,4,4,3,3,3,0,2,2,4,3,3,1,3,3,3,1,1,0,3,1,4,3,2,3,4,4,2,4,4,4,3,4,4,3,2,4,4,3,1,3,3,1,3,3,0,4,1,0,2,2,1,4,3,2,3,3,5,4,3,3,5,4,4,3,3,0,4,0,3,2,2,4,4,}, -{ 0,2,0,1,0,0,0,0,0,1,2,1,3,0,0,0,0,0,2,0,1,2,1,0,0,1,0,0,0,0,3,0,0,1,0,1,1,3,1,0,0,0,1,1,0,1,1,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,1,2,2,0,3,4,0,0,0,1,1,0,0,1,0,0,0,0,0,1,1,}, -{ 0,1,0,0,0,1,0,0,0,0,4,0,4,1,4,0,3,0,4,0,3,0,4,0,3,0,3,0,4,1,5,1,4,0,0,3,0,5,0,5,2,0,1,0,0,0,2,1,4,0,1,3,0,0,3,0,0,3,1,1,4,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,}, -{ 1,4,0,5,0,3,0,2,0,3,5,4,4,3,4,3,5,3,4,3,3,0,4,3,3,3,3,3,3,2,4,4,3,1,3,4,4,5,4,4,3,4,4,1,3,5,4,3,3,3,1,2,2,3,3,1,3,1,3,3,3,5,3,3,4,5,0,3,0,3,0,3,4,3,4,4,3,0,3,0,2,4,3,}, -{ 0,1,0,4,0,0,0,0,0,1,4,0,4,1,4,2,4,0,3,0,1,0,1,0,0,0,0,0,2,0,3,1,1,1,0,3,0,0,0,1,2,1,0,0,1,1,1,1,0,1,0,0,0,1,0,0,3,0,0,0,0,3,2,0,2,2,0,1,0,0,0,2,3,2,3,3,0,0,0,0,2,1,0,}, -{ 0,5,1,5,0,3,0,3,0,5,4,4,5,1,5,3,3,0,4,3,4,3,5,3,4,3,3,2,4,3,4,3,3,0,3,3,1,4,4,3,4,4,4,3,4,5,5,3,2,3,1,1,3,3,1,3,1,1,3,3,2,4,5,3,3,5,0,4,0,3,0,4,4,3,5,3,3,0,3,4,0,4,3,}, -{ 0,5,0,5,0,3,0,2,0,4,4,3,5,2,4,3,3,3,4,4,4,3,5,3,5,3,3,1,4,0,4,3,3,0,3,3,0,4,4,4,4,5,4,3,3,5,5,3,2,3,1,2,3,2,0,1,0,0,3,2,2,4,4,3,1,5,0,4,0,3,0,4,3,1,3,2,1,0,3,3,0,3,3,}, -{ 0,4,0,5,0,5,0,4,0,4,5,5,5,3,4,3,3,2,5,4,4,3,5,3,5,3,4,0,4,3,4,4,3,2,4,4,3,4,5,4,4,5,5,0,3,5,5,4,1,3,3,2,3,3,1,3,1,0,4,3,1,4,4,3,4,5,0,4,0,2,0,4,3,4,4,3,3,0,4,0,0,5,5,}, -{ 0,4,0,4,0,5,0,1,1,3,3,4,4,3,4,1,3,0,5,1,3,0,3,1,3,1,1,0,3,0,3,3,4,0,4,3,0,4,4,4,3,4,4,0,3,5,4,1,0,3,0,0,2,3,0,3,1,0,3,1,0,3,2,1,3,5,0,3,0,1,0,3,2,3,3,4,4,0,2,2,0,4,4,}, -{ 2,4,0,5,0,4,0,3,0,4,5,5,4,3,5,3,5,3,5,3,5,2,5,3,4,3,3,4,3,4,5,3,2,1,5,4,3,2,3,4,5,3,4,1,2,5,4,3,0,3,3,0,3,2,0,2,3,0,4,1,0,3,4,3,3,5,0,3,0,1,0,4,5,5,5,4,3,0,4,2,0,3,5,}, -{ 0,5,0,4,0,4,0,2,0,5,4,3,4,3,4,3,3,3,4,3,4,2,5,3,5,3,4,1,4,3,4,4,4,0,3,5,0,4,4,4,4,5,3,1,3,4,5,3,3,3,3,3,3,3,0,2,2,0,3,3,2,4,3,3,3,5,3,4,1,3,3,5,3,2,0,0,0,0,4,3,1,3,3,}, -{ 0,1,0,3,0,3,0,1,0,1,3,3,3,2,3,3,3,0,3,0,0,0,3,1,3,0,0,0,2,2,2,3,0,0,3,2,0,1,2,4,1,3,3,0,0,3,3,3,0,1,0,0,2,1,0,0,3,0,3,1,0,3,0,0,1,3,0,2,0,1,0,3,3,1,3,3,0,0,1,1,0,3,3,}, -{ 0,2,0,3,0,2,1,4,0,2,2,3,1,1,3,1,1,0,2,0,3,1,2,3,1,3,0,0,1,0,4,3,2,3,3,3,1,4,2,3,3,3,3,1,0,3,1,4,0,1,1,0,1,2,0,1,1,0,1,1,0,3,1,3,2,2,0,1,0,0,0,2,3,3,3,1,0,0,0,0,0,2,3,}, -{ 0,5,0,4,0,5,0,2,0,4,5,5,3,3,4,3,3,1,5,4,4,2,4,4,4,3,4,2,4,3,5,5,4,3,3,4,3,3,5,5,4,5,5,1,3,4,5,3,1,4,3,1,3,3,0,3,3,1,4,3,1,4,5,3,3,5,0,4,0,3,0,5,3,3,1,4,3,0,4,0,1,5,3,}, -{ 0,5,0,5,0,4,0,2,0,4,4,3,4,3,3,3,3,3,5,4,4,4,4,4,4,5,3,3,5,2,4,4,4,3,4,4,3,3,4,4,5,5,3,3,4,3,4,3,3,4,3,3,3,3,1,2,2,1,4,3,3,5,4,4,3,4,0,4,0,3,0,4,4,4,4,4,1,0,4,2,0,2,4,}, -{ 0,4,0,4,0,3,0,1,0,3,5,2,3,0,3,0,2,1,4,2,3,3,4,1,4,3,3,2,4,1,3,3,3,0,3,3,0,0,3,3,3,5,3,3,3,3,3,2,0,2,0,0,2,0,0,2,0,0,1,0,0,3,1,2,2,3,0,3,0,2,0,4,4,3,3,4,1,0,3,0,0,2,4,}, -{ 0,0,0,4,0,0,0,0,0,0,1,0,1,0,2,0,0,0,0,0,1,0,2,0,1,0,0,0,0,0,3,1,3,0,3,2,0,0,0,1,0,3,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,4,0,2,0,0,0,0,0,0,2,}, -{ 0,2,1,3,0,2,0,2,0,3,3,3,3,1,3,1,3,3,3,3,3,3,4,2,2,1,2,1,4,0,4,3,1,3,3,3,2,4,3,5,4,3,3,3,3,3,3,3,0,1,3,0,2,0,0,1,0,0,1,0,0,4,2,0,2,3,0,3,3,0,3,3,4,2,3,1,4,0,1,2,0,2,3,}, -{ 0,3,0,3,0,1,0,3,0,2,3,3,3,0,3,1,2,0,3,3,2,3,3,2,3,2,3,1,3,0,4,3,2,0,3,3,1,4,3,3,2,3,4,3,1,3,3,1,1,0,1,1,0,1,0,1,0,1,0,0,0,4,1,1,0,3,0,3,1,0,2,3,3,3,3,3,1,0,0,2,0,3,3,}, -{ 0,0,0,0,0,0,0,0,0,0,3,0,2,0,3,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,3,0,3,0,3,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,2,0,2,3,0,0,0,0,0,0,0,0,3,}, -{ 0,2,0,3,1,3,0,3,0,2,3,3,3,1,3,1,3,1,3,1,3,3,3,1,3,0,2,3,1,1,4,3,3,2,3,3,1,2,2,4,1,3,3,0,1,4,2,3,0,1,3,0,3,0,0,1,3,0,2,0,0,3,3,2,1,3,0,3,0,2,0,3,4,4,4,3,1,0,3,0,0,3,3,}, -{ 0,2,0,1,0,2,0,0,0,1,3,2,2,1,3,0,1,1,3,0,3,2,3,1,2,0,2,0,1,1,3,3,3,0,3,3,1,1,2,3,2,3,3,1,2,3,2,0,0,1,0,0,0,0,0,0,3,0,1,0,0,2,1,2,1,3,0,3,0,0,0,3,4,4,4,3,2,0,2,0,0,2,4,}, -{ 0,0,0,1,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,2,2,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,3,1,0,0,0,0,0,0,0,3,}, -{ 0,3,0,3,0,2,0,3,0,3,3,3,2,3,2,2,2,0,3,1,3,3,3,2,3,3,0,0,3,0,3,2,2,0,2,3,1,4,3,4,3,3,2,3,1,5,4,4,0,3,1,2,1,3,0,3,1,1,2,0,2,3,1,3,1,3,0,3,0,1,0,3,3,4,4,2,1,0,2,1,0,2,4,}, -{ 0,1,0,3,0,1,0,2,0,1,4,2,5,1,4,0,2,0,2,1,3,1,4,0,2,1,0,0,2,1,4,1,1,0,3,3,0,5,1,3,2,3,3,1,0,3,2,3,0,1,0,0,0,0,0,0,1,0,0,0,0,4,0,1,0,3,0,2,0,1,0,3,3,3,4,3,3,0,0,0,0,2,3,}, -{ 0,0,0,1,0,0,0,0,0,0,2,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,1,0,0,1,0,0,0,0,0,3,}, -{ 0,1,0,3,0,4,0,3,0,2,4,3,1,0,3,2,2,1,3,1,2,2,3,1,1,1,2,1,3,0,1,2,0,1,3,2,1,3,0,5,5,1,0,0,1,3,2,1,0,3,0,0,1,0,0,0,0,0,3,4,0,1,1,1,3,2,0,2,0,1,0,2,3,3,1,2,3,0,1,0,1,0,4,}, -{ 0,0,0,1,0,3,0,3,0,2,2,1,0,0,4,0,3,0,3,1,3,0,3,0,3,0,1,0,3,0,3,1,3,0,3,3,0,0,1,2,1,1,1,0,1,2,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,2,2,1,2,0,0,2,0,0,0,0,2,3,3,3,3,0,0,0,0,1,4,}, -{ 0,0,0,3,0,3,0,0,0,0,3,1,1,0,3,0,1,0,2,0,1,0,0,0,0,0,0,0,1,0,3,0,2,0,2,3,0,0,2,2,3,1,2,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,2,0,0,0,0,2,3,}, -{ 2,4,0,5,0,5,0,4,0,3,4,3,3,3,4,3,3,3,4,3,4,4,5,4,5,5,5,2,3,0,5,5,4,1,5,4,3,1,5,4,3,4,4,3,3,4,3,3,0,3,2,0,2,3,0,3,0,0,3,3,0,5,3,2,3,3,0,3,0,3,0,3,4,5,4,5,3,0,4,3,0,3,4,}, -{ 0,3,0,3,0,3,0,3,0,3,3,4,3,2,3,2,3,0,4,3,3,3,3,3,3,3,3,0,3,2,4,3,3,1,3,4,3,4,4,4,3,4,4,3,2,4,4,1,0,2,0,0,1,1,0,2,0,0,3,1,0,5,3,2,1,3,0,3,0,1,2,4,3,2,4,3,3,0,3,2,0,4,4,}, -{ 0,3,0,3,0,1,0,0,0,1,4,3,3,2,3,1,3,1,4,2,3,2,4,2,3,4,3,0,2,2,3,3,3,0,3,3,3,0,3,4,1,3,3,0,3,4,3,3,0,1,1,0,1,0,0,0,4,0,3,0,0,3,1,2,1,3,0,4,0,1,0,4,3,3,4,3,3,0,2,0,0,3,3,}, -{ 0,3,0,4,0,1,0,3,0,3,4,3,3,0,3,3,3,1,3,1,3,3,4,3,3,3,0,0,3,1,5,3,3,1,3,3,2,5,4,3,3,4,5,3,2,5,3,4,0,1,0,0,0,0,0,2,0,0,1,1,0,4,2,2,1,3,0,3,0,2,0,4,4,3,5,3,2,0,1,1,0,3,4,}, -{ 0,5,0,4,0,5,0,2,0,4,4,3,3,2,3,3,3,1,4,3,4,1,5,3,4,3,4,0,4,2,4,3,4,1,5,4,0,4,4,4,4,5,4,1,3,5,4,2,1,4,1,1,3,2,0,3,1,0,3,2,1,4,3,3,3,4,0,4,0,3,0,4,4,4,3,3,3,0,4,2,0,3,4,}, -{ 1,4,0,4,0,3,0,1,0,3,3,3,1,1,3,3,2,2,3,3,1,0,3,2,2,1,2,0,3,1,2,1,2,0,3,2,0,2,2,3,3,4,3,0,3,3,1,2,0,1,1,3,1,2,0,0,3,0,1,1,0,3,2,2,3,3,0,3,0,0,0,2,3,3,4,3,3,0,1,0,0,1,4,}, -{ 0,4,0,4,0,4,0,0,0,3,4,4,3,1,4,2,3,2,3,3,3,1,4,3,4,0,3,0,4,2,3,3,2,2,5,4,2,1,3,4,3,4,3,1,3,3,4,2,0,2,1,0,3,3,0,0,2,0,3,1,0,4,4,3,4,3,0,4,0,1,0,2,4,4,4,4,4,0,3,2,0,3,3,}, -{ 0,0,0,1,0,4,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,3,2,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,2,}, -{ 0,2,0,3,0,4,0,4,0,1,3,3,3,0,4,0,2,1,2,1,1,1,2,0,3,1,1,0,1,0,3,1,0,0,3,3,2,0,1,1,0,0,0,0,0,1,0,2,0,2,2,0,3,1,0,0,1,0,1,1,0,1,2,0,3,0,0,0,0,1,0,0,3,3,4,3,1,0,1,0,3,0,2,}, -{ 0,0,0,3,0,5,0,0,0,0,1,0,2,0,3,1,0,1,3,0,0,0,2,0,0,0,1,0,0,0,1,1,0,0,4,0,0,0,2,3,0,1,4,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,1,0,0,0,0,0,0,0,2,0,0,3,0,0,0,0,0,3,}, -{ 0,2,0,5,0,5,0,1,0,2,4,3,3,2,5,1,3,2,3,3,3,0,4,1,2,0,3,0,4,0,2,2,1,1,5,3,0,0,1,4,2,3,2,0,3,3,3,2,0,2,4,1,1,2,0,1,1,0,3,1,0,1,3,1,2,3,0,2,0,0,0,1,3,5,4,4,4,0,3,0,0,1,3,}, -{ 0,4,0,5,0,4,0,4,0,4,5,4,3,3,4,3,3,3,4,3,4,4,5,3,4,5,4,2,4,2,3,4,3,1,4,4,1,3,5,4,4,5,5,4,4,5,5,5,2,3,3,1,4,3,1,3,3,0,3,3,1,4,3,4,4,4,0,3,0,4,0,3,3,4,4,5,0,0,4,3,0,4,5,}, -{ 0,4,0,4,0,3,0,3,0,3,4,4,4,3,3,2,4,3,4,3,4,3,5,3,4,3,2,1,4,2,4,4,3,1,3,4,2,4,5,5,3,4,5,4,1,5,4,3,0,3,2,2,3,2,1,3,1,0,3,3,3,5,3,3,3,5,4,4,2,3,3,4,3,3,3,2,1,0,3,2,1,4,3,}, -{ 0,4,0,5,0,4,0,3,0,3,5,5,3,2,4,3,4,0,5,4,4,1,4,4,4,3,3,3,4,3,5,5,2,3,3,4,1,2,5,5,3,5,5,2,3,5,5,4,0,3,2,0,3,3,1,1,5,1,4,1,0,4,3,2,3,5,0,4,0,3,0,5,4,3,4,3,0,0,4,1,0,4,4,}, -{ 1,3,0,4,0,2,0,2,0,2,5,5,3,3,3,3,3,0,4,2,3,4,4,4,3,4,0,0,3,4,5,4,3,3,3,3,2,5,5,4,5,5,5,4,3,5,5,5,1,3,1,0,1,0,0,3,2,0,4,2,0,5,2,3,2,4,1,3,0,3,0,4,5,4,5,4,3,0,4,2,0,5,4,}, -{ 0,3,0,4,0,5,0,3,0,3,4,4,3,2,3,2,3,3,3,3,3,2,4,3,3,2,2,0,3,3,3,3,3,1,3,3,3,0,4,4,3,4,4,1,1,4,4,2,0,3,1,0,1,1,0,4,1,0,2,3,1,3,3,1,3,4,0,3,0,1,0,3,1,3,0,0,1,0,2,0,0,4,4,}, -{ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}, -{ 0,3,0,3,0,2,0,3,0,1,5,4,3,3,3,1,4,2,1,2,3,4,4,2,4,4,5,0,3,1,4,3,4,0,4,3,3,3,2,3,2,5,3,4,3,2,2,3,0,0,3,0,2,1,0,1,2,0,0,0,0,2,1,1,3,1,0,2,0,4,0,3,4,4,4,5,2,0,2,0,0,1,3,}, -{ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,1,0,0,1,1,0,0,0,4,2,1,1,0,1,0,3,2,0,0,3,1,1,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,1,0,0,0,2,0,0,0,1,4,0,4,2,1,0,0,0,0,0,1,}, -{ 0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0,3,1,0,0,0,2,0,2,1,0,0,1,2,1,0,1,1,0,0,3,0,0,0,0,0,0,0,0,0,0,0,1,3,1,0,0,0,0,0,1,0,0,2,1,0,0,0,0,0,0,0,0,2,}, -{ 0,4,0,4,0,4,0,3,0,4,4,3,4,2,4,3,2,0,4,4,4,3,5,3,5,3,3,2,4,2,4,3,4,3,1,4,0,2,3,4,4,4,3,3,3,4,4,4,3,4,1,3,4,3,2,1,2,1,3,3,3,4,4,3,3,5,0,4,0,3,0,4,3,3,3,2,1,0,3,0,0,3,3,}, -{ 0,4,0,3,0,3,0,3,0,3,5,5,3,3,3,3,4,3,4,3,3,3,4,4,4,3,3,3,3,4,3,5,3,3,1,3,2,4,5,5,5,5,4,3,4,5,5,3,2,2,3,3,3,3,2,3,3,1,2,3,2,4,3,3,3,4,0,4,0,2,0,4,3,2,2,1,2,0,3,0,0,4,1,}, -}; - -#define MINIMUM_DATA_THRESHOLD 4 - -void JapaneseContextAnalysis::HandleData(const char* aBuf, PRUint32 aLen) -{ - PRUint32 charLen; - PRInt32 order; - PRUint32 i; - - if (mDone) - return; - - //The buffer we got is byte oriented, and a character may span in more than one - //buffers. In case the last one or two byte in last buffer is not complete, we - //record how many byte needed to complete that character and skip these bytes here. - //We can choose to record those bytes as well and analyse the character once it - //is complete, but since a character will not make much difference, by simply skipping - //this character will simply our logic and improve performance. - for (i = mNeedToSkipCharNum; i < aLen; ) - { - order = GetOrder(aBuf+i, &charLen); - i+= charLen; - if (i > aLen){ - mNeedToSkipCharNum = i - aLen; - mLastCharOrder = -1; - } - else - { - if (order != -1 && mLastCharOrder != -1) - { - mTotalRel ++; - if (mTotalRel > MAX_REL_THRESHOLD) - { - mDone = PR_TRUE; - break; - } - mRelSample[jp2CharContext[mLastCharOrder][order]]++; - } - mLastCharOrder = order; - } - } - - return; -} - -void JapaneseContextAnalysis::Reset(PRBool aIsPreferredLanguage) -{ - mTotalRel = 0; - for (PRUint32 i = 0; i < NUM_OF_CATEGORY; i++) - mRelSample[i] = 0; - mNeedToSkipCharNum = 0; - mLastCharOrder = -1; - mDone = PR_FALSE; - mDataThreshold = aIsPreferredLanguage ? 0 : MINIMUM_DATA_THRESHOLD; -} -#define DONT_KNOW (float)-1 - -float JapaneseContextAnalysis::GetConfidence(void) -{ - //This is just one way to calculate confidence. It works well for me. - if (mTotalRel > mDataThreshold) - return ((float)(mTotalRel - mRelSample[0]))/mTotalRel; - else - return (float)DONT_KNOW; -} - - -PRInt32 SJISContextAnalysis::GetOrder(const char* str, PRUint32 *charLen) -{ - //find out current char's byte length - if ((unsigned char)*str >= (unsigned char)0x81 && (unsigned char)*str <= (unsigned char)0x9f || - (unsigned char)*str >= (unsigned char)0xe0 && (unsigned char)*str <= (unsigned char)0xfc ) - *charLen = 2; - else - *charLen = 1; - - //return its order if it is hiragana - if (*str == '\202' && - (unsigned char)*(str+1) >= (unsigned char)0x9f && - (unsigned char)*(str+1) <= (unsigned char)0xf1) - return (unsigned char)*(str+1) - (unsigned char)0x9f; - return -1; -} - -PRInt32 EUCJPContextAnalysis::GetOrder(const char* str, PRUint32 *charLen) -{ - //find out current char's byte length - if ((unsigned char)*str == (unsigned char)0x8e || - (unsigned char)*str >= (unsigned char)0xa1 && - (unsigned char)*str <= (unsigned char)0xfe) - *charLen = 2; - else if ((unsigned char)*str == (unsigned char)0x8f) - *charLen = 3; - else - *charLen = 1; - - //return its order if it is hiragana - if ((unsigned char)*str == (unsigned char)0xa4 && - (unsigned char)*(str+1) >= (unsigned char)0xa1 && - (unsigned char)*(str+1) <= (unsigned char)0xf3) - return (unsigned char)*(str+1) - (unsigned char)0xa1; - return -1; -} diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/JpCntx.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/JpCntx.h deleted file mode 100644 index fe8fcb8..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/JpCntx.h +++ /dev/null @@ -1,140 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef __JPCNTX_H__ -#define __JPCNTX_H__ - -#define NUM_OF_CATEGORY 6 - -#include "nscore.h" - -#define ENOUGH_REL_THRESHOLD 100 -#define MAX_REL_THRESHOLD 1000 - -//hiragana frequency category table -extern const PRUint8 jp2CharContext[83][83]; - -class JapaneseContextAnalysis -{ -public: - JapaneseContextAnalysis() {Reset(PR_FALSE);} - - void HandleData(const char* aBuf, PRUint32 aLen); - - void HandleOneChar(const char* aStr, PRUint32 aCharLen) - { - PRInt32 order; - - //if we received enough data, stop here - if (mTotalRel > MAX_REL_THRESHOLD) mDone = PR_TRUE; - if (mDone) return; - - //Only 2-bytes characters are of our interest - order = (aCharLen == 2) ? GetOrder(aStr) : -1; - if (order != -1 && mLastCharOrder != -1) - { - mTotalRel++; - //count this sequence to its category counter - mRelSample[jp2CharContext[mLastCharOrder][order]]++; - } - mLastCharOrder = order; - } - - float GetConfidence(void); - void Reset(PRBool aIsPreferredLanguage); - void SetOpion(){} - PRBool GotEnoughData() {return mTotalRel > ENOUGH_REL_THRESHOLD;} - -protected: - virtual PRInt32 GetOrder(const char* str, PRUint32 *charLen) = 0; - virtual PRInt32 GetOrder(const char* str) = 0; - - //category counters, each integer counts sequences in its category - PRUint32 mRelSample[NUM_OF_CATEGORY]; - - //total sequence received - PRUint32 mTotalRel; - - //Number of sequences needed to trigger detection - PRUint32 mDataThreshold; - - //The order of previous char - PRInt32 mLastCharOrder; - - //if last byte in current buffer is not the last byte of a character, we - //need to know how many byte to skip in next buffer. - PRUint32 mNeedToSkipCharNum; - - //If this flag is set to PR_TRUE, detection is done and conclusion has been made - PRBool mDone; -}; - - -class SJISContextAnalysis : public JapaneseContextAnalysis -{ - //SJISContextAnalysis(){}; -protected: - PRInt32 GetOrder(const char* str, PRUint32 *charLen); - - PRInt32 GetOrder(const char* str) - { - //We only interested in Hiragana, so first byte is '\202' - if (*str == '\202' && - (unsigned char)*(str+1) >= (unsigned char)0x9f && - (unsigned char)*(str+1) <= (unsigned char)0xf1) - return (unsigned char)*(str+1) - (unsigned char)0x9f; - return -1; - } -}; - -class EUCJPContextAnalysis : public JapaneseContextAnalysis -{ -protected: - PRInt32 GetOrder(const char* str, PRUint32 *charLen); - PRInt32 GetOrder(const char* str) - //We only interested in Hiragana, so first byte is '\244' - { - if (*str == '\244' && - (unsigned char)*(str+1) >= (unsigned char)0xa1 && - (unsigned char)*(str+1) <= (unsigned char)0xf3) - return (unsigned char)*(str+1) - (unsigned char)0xa1; - return -1; - } -}; - -#endif /* __JPCNTX_H__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangBulgarianModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangBulgarianModel.cpp deleted file mode 100644 index 4bfbad7..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangBulgarianModel.cpp +++ /dev/null @@ -1,247 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" -/**************************************************************** -255: Control characters that usually does not exist in any text -254: Carriage/Return -253: symbol (punctuation) that does not belong to word -252: 0 - 9 - -*****************************************************************/ - -//Character Mapping Table: -//this talbe is modified base on win1251BulgarianCharToOrderMap, so -//only number <64 is sure valid - -static const unsigned char Latin5_BulgarianCharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, //00 -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, //10 -+253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, //20 -252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, //30 -253, 77, 90, 99,100, 72,109,107,101, 79,185, 81,102, 76, 94, 82, //40 -110,186,108, 91, 74,119, 84, 96,111,187,115,253,253,253,253,253, //50 -253, 65, 69, 70, 66, 63, 68,112,103, 92,194,104, 95, 86, 87, 71, //60 -116,195, 85, 93, 97,113,196,197,198,199,200,253,253,253,253,253, //70 -194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209, //80 -210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225, //90 - 81,226,227,228,229,230,105,231,232,233,234,235,236, 45,237,238, //a0 - 31, 32, 35, 43, 37, 44, 55, 47, 40, 59, 33, 46, 38, 36, 41, 30, //b0 - 39, 28, 34, 51, 48, 49, 53, 50, 54, 57, 61,239, 67,240, 60, 56, //c0 - 1, 18, 9, 20, 11, 3, 23, 15, 2, 26, 12, 10, 14, 6, 4, 13, //d0 - 7, 8, 5, 19, 29, 25, 22, 21, 27, 24, 17, 75, 52,241, 42, 16, //e0 - 62,242,243,244, 58,245, 98,246,247,248,249,250,251, 91,252,253, //f0 -}; - -static const unsigned char win1251BulgarianCharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, //00 -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, //10 -+253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, //20 -252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, //30 -253, 77, 90, 99,100, 72,109,107,101, 79,185, 81,102, 76, 94, 82, //40 -110,186,108, 91, 74,119, 84, 96,111,187,115,253,253,253,253,253, //50 -253, 65, 69, 70, 66, 63, 68,112,103, 92,194,104, 95, 86, 87, 71, //60 -116,195, 85, 93, 97,113,196,197,198,199,200,253,253,253,253,253, //70 -206,207,208,209,210,211,212,213,120,214,215,216,217,218,219,220, //80 -221, 78, 64, 83,121, 98,117,105,222,223,224,225,226,227,228,229, //90 - 88,230,231,232,233,122, 89,106,234,235,236,237,238, 45,239,240, //a0 - 73, 80,118,114,241,242,243,244,245, 62, 58,246,247,248,249,250, //b0 - 31, 32, 35, 43, 37, 44, 55, 47, 40, 59, 33, 46, 38, 36, 41, 30, //c0 - 39, 28, 34, 51, 48, 49, 53, 50, 54, 57, 61,251, 67,252, 60, 56, //d0 - 1, 18, 9, 20, 11, 3, 23, 15, 2, 26, 12, 10, 14, 6, 4, 13, //e0 - 7, 8, 5, 19, 29, 25, 22, 21, 27, 24, 17, 75, 52,253, 42, 16, //f0 -}; - -//Model Table: -//total sequences: 100% -//first 512 sequences: 96.9392% -//first 1024 sequences:3.0618% -//rest sequences: 0.2992% -//negative sequences: 0.0020% -static const PRUint8 BulgarianLangModel[] = -{ -0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,3,3,3,3,3, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,2,2,3,2,2,1,2,2, -3,1,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,3,3,3,3,3,3,0,3,0,1, -0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,2,3,3,3,3,3,3,3,3,0,3,1,0, -0,1,0,0,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -3,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,1,3,2,3,3,3,3,3,3,3,3,0,3,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,1,3,2,3,3,3,3,3,3,3,3,0,3,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,2,3,2,2,1,3,3,3,3,2,2,2,1,1,2,0,1,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,3,3,2,3,2,2,3,3,1,1,2,3,3,2,3,3,3,3,2,1,2,0,2,0,3,0,0, -0,0,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,3,3,1,3,3,3,3,3,2,3,2,3,3,3,3,3,2,3,3,1,3,0,3,0,2,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,3,3,3,1,3,3,2,3,3,3,1,3,3,2,3,2,2,2,0,0,2,0,2,0,2,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,3,3,3,3,0,3,3,3,2,2,3,3,3,1,2,2,3,2,1,1,2,0,2,0,0,0,0, -1,0,0,0,0,0,0,0,0,0,2,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,3,3,2,3,3,1,2,3,2,2,2,3,3,3,3,3,2,2,3,1,2,0,2,1,2,0,0, -0,0,0,0,0,0,0,0,0,0,3,0,0,1,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,1,3,3,3,3,3,2,3,3,3,2,3,3,2,3,2,2,2,3,1,2,0,1,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,3,3,3,3,3,3,1,1,1,2,2,1,3,1,3,2,2,3,0,0,1,0,1,0,1,0,0, -0,0,0,1,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,2,2,3,2,2,3,1,2,1,1,1,2,3,1,3,1,2,2,0,1,1,1,1,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,1,3,2,2,3,3,1,2,3,1,1,3,3,3,3,1,2,2,1,1,1,0,2,0,2,0,1, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,2,2,3,3,3,2,2,1,1,2,0,2,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1, -3,0,1,2,1,3,3,2,3,3,3,3,3,2,3,2,1,0,3,1,2,1,2,1,2,3,2,1,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,1,1,2,3,3,3,3,3,3,3,3,3,3,3,3,0,0,3,1,3,3,2,3,3,2,2,2,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,3,0,3,3,3,3,3,2,1,1,2,1,3,3,0,3,1,1,1,1,3,2,0,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1, -3,3,2,2,2,3,3,3,3,3,3,3,3,3,3,3,1,1,3,1,3,3,2,3,2,2,2,3,0,2,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,2,3,3,2,2,3,2,1,1,1,1,1,3,1,3,1,1,0,0,0,1,0,0,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,2,3,2,0,3,2,0,3,0,2,0,0,2,1,3,1,0,0,1,0,0,0,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,2,1,1,1,1,2,1,1,2,1,1,1,2,2,1,2,1,1,1,0,1,1,0,1,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,2,1,3,1,1,2,1,3,2,1,1,0,1,2,3,2,1,1,1,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,3,2,2,1,0,1,0,0,1,0,0,0,2,1,0,3,0,0,1,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,2,3,2,3,3,1,3,2,1,1,1,2,1,1,2,1,3,0,1,0,0,0,1,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,1,1,2,2,3,3,2,3,2,2,2,3,1,2,2,1,1,2,1,1,2,2,0,1,1,0,1,0,2,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,1,3,1,0,2,2,1,3,2,1,0,0,2,0,2,0,1,0,0,0,0,0,0,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,3,1,2,0,2,3,1,2,3,2,0,1,3,1,2,1,1,1,0,0,1,0,0,2,2,2,3, -2,2,2,2,1,2,1,1,2,2,1,1,2,0,1,1,1,0,0,1,1,0,0,1,1,0,0,0,1,1,0,1, -3,3,3,3,3,2,1,2,2,1,2,0,2,0,1,0,1,2,1,2,1,1,0,0,0,1,0,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1, -3,3,2,3,3,1,1,3,1,0,3,2,1,0,0,0,1,2,0,2,0,1,0,0,0,1,0,1,2,1,2,2, -1,1,1,1,1,1,1,2,2,2,1,1,1,1,1,1,1,0,1,2,1,1,1,0,0,0,0,0,1,1,0,0, -3,1,0,1,0,2,3,2,2,2,3,2,2,2,2,2,1,0,2,1,2,1,1,1,0,1,2,1,2,2,2,1, -1,1,2,2,2,2,1,2,1,1,0,1,2,1,2,2,2,1,1,1,0,1,1,1,1,2,0,1,0,0,0,0, -2,3,2,3,3,0,0,2,1,0,2,1,0,0,0,0,2,3,0,2,0,0,0,0,0,1,0,0,2,0,1,2, -2,1,2,1,2,2,1,1,1,2,1,1,1,0,1,2,2,1,1,1,1,1,0,1,1,1,0,0,1,2,0,0, -3,3,2,2,3,0,2,3,1,1,2,0,0,0,1,0,0,2,0,2,0,0,0,1,0,1,0,1,2,0,2,2, -1,1,1,1,2,1,0,1,2,2,2,1,1,1,1,1,1,1,0,1,1,1,0,0,0,0,0,0,1,1,0,0, -2,3,2,3,3,0,0,3,0,1,1,0,1,0,0,0,2,2,1,2,0,0,0,0,0,0,0,0,2,0,1,2, -2,2,1,1,1,1,1,2,2,2,1,0,2,0,1,0,1,0,0,1,0,1,0,0,1,0,0,0,0,1,0,0, -3,3,3,3,2,2,2,2,2,0,2,1,1,1,1,2,1,2,1,1,0,2,0,1,0,1,0,0,2,0,1,2, -1,1,1,1,1,1,1,2,2,1,1,0,2,0,1,0,2,0,0,1,1,1,0,0,2,0,0,0,1,1,0,0, -2,3,3,3,3,1,0,0,0,0,0,0,0,0,0,0,2,0,0,1,1,0,0,0,0,0,0,1,2,0,1,2, -2,2,2,1,1,2,1,1,2,2,2,1,2,0,1,1,1,1,1,1,0,1,1,1,1,0,0,1,1,1,0,0, -2,3,3,3,3,0,2,2,0,2,1,0,0,0,1,1,1,2,0,2,0,0,0,3,0,0,0,0,2,0,2,2, -1,1,1,2,1,2,1,1,2,2,2,1,2,0,1,1,1,0,1,1,1,1,0,2,1,0,0,0,1,1,0,0, -2,3,3,3,3,0,2,1,0,0,2,0,0,0,0,0,1,2,0,2,0,0,0,0,0,0,0,0,2,0,1,2, -1,1,1,2,1,1,1,1,2,2,2,0,1,0,1,1,1,0,0,1,1,1,0,0,1,0,0,0,0,1,0,0, -3,3,2,2,3,0,1,0,1,0,0,0,0,0,0,0,1,1,0,3,0,0,0,0,0,0,0,0,1,0,2,2, -1,1,1,1,1,2,1,1,2,2,1,2,2,1,0,1,1,1,1,1,0,1,0,0,1,0,0,0,1,1,0,0, -3,1,0,1,0,2,2,2,2,3,2,1,1,1,2,3,0,0,1,0,2,1,1,0,1,1,1,1,2,1,1,1, -1,2,2,1,2,1,2,2,1,1,0,1,2,1,2,2,1,1,1,0,0,1,1,1,2,1,0,1,0,0,0,0, -2,1,0,1,0,3,1,2,2,2,2,1,2,2,1,1,1,0,2,1,2,2,1,1,2,1,1,0,2,1,1,1, -1,2,2,2,2,2,2,2,1,2,0,1,1,0,2,1,1,1,1,1,0,0,1,1,1,1,0,1,0,0,0,0, -2,1,1,1,1,2,2,2,2,1,2,2,2,1,2,2,1,1,2,1,2,3,2,2,1,1,1,1,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,3,2,0,1,2,0,1,2,1,1,0,1,0,1,2,1,2,0,0,0,1,1,0,0,0,1,0,0,2, -1,1,0,0,1,1,0,1,1,1,1,0,2,0,1,1,1,0,0,1,1,0,0,0,0,1,0,0,0,1,0,0, -2,0,0,0,0,1,2,2,2,2,2,2,2,1,2,1,1,1,1,1,1,1,0,1,1,1,1,1,2,1,1,1, -1,2,2,2,2,1,1,2,1,2,1,1,1,0,2,1,2,1,1,1,0,2,1,1,1,1,0,1,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0, -1,1,0,1,0,1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,3,2,0,0,0,0,1,0,0,0,0,0,0,1,1,0,2,0,0,0,0,0,0,0,0,1,0,1,2, -1,1,1,1,1,1,0,0,2,2,2,2,2,0,1,1,0,1,1,1,1,1,0,0,1,0,0,0,1,1,0,1, -2,3,1,2,1,0,1,1,0,2,2,2,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0,1,0,1,2, -1,1,1,1,2,1,1,1,1,1,1,1,1,0,1,1,0,1,0,1,0,1,0,0,1,0,0,0,0,1,0,0, -2,2,2,2,2,0,0,2,0,0,2,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,2,0,2,2, -1,1,1,1,1,0,0,1,2,1,1,0,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0, -1,2,2,2,2,0,0,2,0,1,1,0,0,0,1,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,1,1, -0,0,0,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0, -1,2,2,3,2,0,0,1,0,0,1,0,0,0,0,0,0,1,0,2,0,0,0,1,0,0,0,0,0,0,0,2, -1,1,0,0,1,0,0,0,1,1,0,0,1,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0, -2,1,2,2,2,1,2,1,2,2,1,1,2,1,1,1,0,1,1,1,1,2,0,1,0,1,1,1,1,0,1,1, -1,1,2,1,1,1,1,1,1,0,0,1,2,1,1,1,1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0, -1,0,0,1,3,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,2,1,0,0,1,0,2,0,0,0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,2,0,0,1, -0,2,0,1,0,0,1,1,2,0,1,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0, -1,2,2,2,2,0,1,1,0,2,1,0,1,1,1,0,0,1,0,2,0,1,0,0,0,0,0,0,0,0,0,1, -0,1,0,0,1,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,2,2,0,0,1,0,0,0,1,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1, -0,1,0,1,1,1,0,0,1,1,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0, -2,0,1,0,0,1,2,1,1,1,1,1,1,2,2,1,0,0,1,0,1,0,0,0,0,1,1,1,1,0,0,0, -1,1,2,1,1,1,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,1,2,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1, -0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0, -0,1,1,0,1,1,1,0,0,1,0,0,1,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0, -1,0,1,0,0,1,1,1,1,1,1,1,1,1,1,1,0,0,1,0,2,0,0,2,0,1,0,0,1,0,0,1, -1,1,0,0,1,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0, -1,1,1,1,1,1,1,2,0,0,0,0,0,0,2,1,0,1,1,0,0,1,1,1,0,1,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,0,1,1,1,1,1,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -}; - -const SequenceModel Latin5BulgarianModel = -{ - Latin5_BulgarianCharToOrderMap, - BulgarianLangModel, - (float)0.969392, - PR_FALSE, - "ISO-8859-5", - "Bulgarian" -}; - -const SequenceModel Win1251BulgarianModel = -{ - win1251BulgarianCharToOrderMap, - BulgarianLangModel, - (float)0.969392, - PR_FALSE, - "WINDOWS-1251", - "Bulgarian" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangCyrillicModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangCyrillicModel.cpp deleted file mode 100644 index 6f54f19..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangCyrillicModel.cpp +++ /dev/null @@ -1,361 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - - - -//KOI8-R language model -//Character Mapping Table: -static const unsigned char KOI8R_CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, //00 -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, //10 -+253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, //20 -252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, //30 -253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, //40 -155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, //50 -253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, //60 - 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, //70 -191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206, //80 -207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222, //90 -223,224,225, 68,226,227,228,229,230,231,232,233,234,235,236,237, //a0 -238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253, //b0 - 27, 3, 21, 28, 13, 2, 39, 19, 26, 4, 23, 11, 8, 12, 5, 1, //c0 - 15, 16, 9, 7, 6, 14, 24, 10, 17, 18, 20, 25, 30, 29, 22, 54, //d0 - 59, 37, 44, 58, 41, 48, 53, 46, 55, 42, 60, 36, 49, 38, 31, 34, //e0 - 35, 43, 45, 32, 40, 52, 56, 33, 61, 62, 51, 57, 47, 63, 50, 70, //f0 -}; - -static const unsigned char win1251_CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, //00 -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, //10 -+253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, //20 -252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, //30 -253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, //40 -155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, //50 -253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, //60 - 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, //70 -191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206, -207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222, -223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238, -239,240,241,242,243,244,245,246, 68,247,248,249,250,251,252,253, - 37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35, - 45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43, - 3, 21, 10, 19, 13, 2, 24, 20, 4, 23, 11, 8, 12, 5, 1, 15, - 9, 7, 6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16, -}; - -static const unsigned char latin5_CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, //00 -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, //10 -+253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, //20 -252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, //30 -253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, //40 -155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, //50 -253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, //60 - 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, //70 -191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206, -207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222, -223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238, - 37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35, - 45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43, - 3, 21, 10, 19, 13, 2, 24, 20, 4, 23, 11, 8, 12, 5, 1, 15, - 9, 7, 6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16, -239, 68,240,241,242,243,244,245,246,247,248,249,250,251,252,255, -}; - -static const unsigned char macCyrillic_CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, //00 -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, //10 -+253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, //20 -252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, //30 -253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, //40 -155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, //50 -253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, //60 - 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, //70 - 37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35, - 45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43, -191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206, -207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222, -223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238, -239,240,241,242,243,244,245,246,247,248,249,250,251,252, 68, 16, - 3, 21, 10, 19, 13, 2, 24, 20, 4, 23, 11, 8, 12, 5, 1, 15, - 9, 7, 6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27,255, -}; - -static const unsigned char IBM855_CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, //00 -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, //10 -+253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, //20 -252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, //30 -253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, //40 -155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, //50 -253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, //60 - 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, //70 -191,192,193,194, 68,195,196,197,198,199,200,201,202,203,204,205, -206,207,208,209,210,211,212,213,214,215,216,217, 27, 59, 54, 70, - 3, 37, 21, 44, 28, 58, 13, 41, 2, 48, 39, 53, 19, 46,218,219, -220,221,222,223,224, 26, 55, 4, 42,225,226,227,228, 23, 60,229, -230,231,232,233,234,235, 11, 36,236,237,238,239,240,241,242,243, - 8, 49, 12, 38, 5, 31, 1, 34, 15,244,245,246,247, 35, 16,248, - 43, 9, 45, 7, 32, 6, 40, 14, 52, 24, 56, 10, 33, 17, 61,249, -250, 18, 62, 20, 51, 25, 57, 30, 47, 29, 63, 22, 50,251,252,255, -}; - -static const unsigned char IBM866_CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, //00 -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, //10 -+253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, //20 -252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, //30 -253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, //40 -155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, //50 -253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, //60 - 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, //70 - 37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35, - 45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43, - 3, 21, 10, 19, 13, 2, 24, 20, 4, 23, 11, 8, 12, 5, 1, 15, -191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206, -207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222, -223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238, - 9, 7, 6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16, -239, 68,240,241,242,243,244,245,246,247,248,249,250,251,252,255, -}; - -//Model Table: -//total sequences: 100% -//first 512 sequences: 97.6601% -//first 1024 sequences: 2.3389% -//rest sequences: 0.1237% -//negative sequences: 0.0009% -static const PRUint8 RussianLangModel[] = -{ -0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,1,3,3,3,3,1,3,3,3,2,3,2,3,3, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,2,2,2,2,2,0,0,2, -3,3,3,2,3,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,3,2,3,2,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,2,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,2,3,3,1,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,3,3,3,3,3,3,3,3,3,3,3,2,1, -0,0,0,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,3,3,3,2,1, -0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,2,2,2,3,1,3,3,1,3,3,3,3,2,2,3,0,2,2,2,3,3,2,1,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,3,3,3,3,2,2,3,2,3,3,3,2,1,2,2,0,1,2,2,2,2,2,2,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,3,0,2,2,3,3,2,1,2,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,3,1,2,3,2,2,3,2,3,3,3,3,2,2,3,0,3,2,2,3,1,1,1,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,2,2,3,3,3,3,3,2,3,3,3,3,2,2,2,0,3,3,3,2,2,2,2,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,2,3,2,3,3,3,3,3,3,2,3,2,2,0,1,3,2,1,2,2,1,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,2,1,1,3,0,1,1,1,1,2,1,1,0,2,2,2,1,2,0,1,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,3,2,2,2,2,1,3,2,3,2,3,2,1,2,2,0,1,1,2,1,2,1,2,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,2,2,3,2,3,3,3,2,2,2,2,0,2,2,2,2,3,1,1,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,2,3,2,2,3,3,3,3,3,3,3,3,3,1,3,2,0,0,3,3,3,3,2,3,3,3,3,2,3,2,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,3,3,2,2,3,3,0,2,1,0,3,2,3,2,3,0,0,1,2,0,0,1,0,1,2,1,1,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,0,2,3,3,3,3,2,3,3,3,3,1,2,2,0,0,2,3,2,2,2,3,2,3,2,2,3,0,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,0,2,3,2,3,0,1,2,3,3,2,0,2,3,0,0,2,3,2,2,0,1,3,1,3,2,2,1,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,1,3,0,2,3,3,3,3,3,3,3,3,2,1,3,2,0,0,2,2,3,3,3,2,3,3,0,2,2,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,2,3,3,2,2,2,3,3,0,0,1,1,1,1,1,2,0,0,1,1,1,1,0,1,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,2,3,3,3,3,3,3,3,0,3,2,3,3,2,3,2,0,2,1,0,1,1,0,1,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,3,3,2,2,2,2,3,1,3,2,3,1,1,2,1,0,2,2,2,2,1,3,1,0, -0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -2,2,3,3,3,3,3,1,2,2,1,3,1,0,3,0,0,3,0,0,0,1,1,0,1,2,1,0,0,0,0,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,2,1,1,3,3,3,2,2,1,2,2,3,1,1,2,0,0,2,2,1,3,0,0,2,1,1,2,1,1,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,3,3,1,2,2,2,1,2,1,3,3,1,1,2,1,2,1,2,2,0,2,0,0,1,1,0,1,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,3,3,2,1,3,2,2,3,2,0,3,2,0,3,0,1,0,1,1,0,0,1,1,1,1,0,1,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,3,3,3,2,2,2,3,3,1,2,1,2,1,0,1,0,1,1,0,1,0,0,2,1,1,1,0,1,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0, -3,1,1,2,1,2,3,3,2,2,1,2,2,3,0,2,1,0,0,2,2,3,2,1,2,2,2,2,2,3,1,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,1,1,0,1,1,2,2,1,1,3,0,0,1,3,1,1,1,0,0,0,1,0,1,1,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,1,3,3,3,2,0,0,0,2,1,0,1,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,1,0,0,2,3,2,2,2,1,2,2,2,1,2,1,0,0,1,1,1,0,2,0,1,1,1,0,0,1,1, -1,0,0,0,0,0,1,2,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,3,0,0,0,0,1,0,0,0,0,3,0,1,2,1,0,0,0,0,0,0,0,1,1,0,0,1,1, -1,0,1,0,1,2,0,0,1,1,2,1,0,1,1,1,1,0,1,1,1,1,0,1,0,0,1,0,0,1,1,0, -2,2,3,2,2,2,3,1,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,0,1,0,1,1,1,0,2,1, -1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,0,1,1,1,0,1,1,0, -3,3,3,2,2,2,2,3,2,2,1,1,2,2,2,2,1,1,3,1,2,1,2,0,0,1,1,0,1,0,2,1, -1,1,1,1,1,2,1,0,1,1,1,1,0,1,0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,1,1,0, -2,0,0,1,0,3,2,2,2,2,1,2,1,2,1,2,0,0,0,2,1,2,2,1,1,2,2,0,1,1,0,2, -1,1,1,1,1,0,1,1,1,2,1,1,1,2,1,0,1,2,1,1,1,1,0,1,1,1,0,0,1,0,0,1, -1,3,2,2,2,1,1,1,2,3,0,0,0,0,2,0,2,2,1,0,0,0,0,0,0,1,0,0,0,0,1,1, -1,0,1,1,0,1,0,1,1,0,1,1,0,2,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0, -2,3,2,3,2,1,2,2,2,2,1,0,0,0,2,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,2,1, -1,1,2,1,0,2,0,0,1,0,1,0,0,1,0,0,1,1,0,1,1,0,0,0,0,0,1,0,0,0,0,0, -3,0,0,1,0,2,2,2,3,2,2,2,2,2,2,2,0,0,0,2,1,2,1,1,1,2,2,0,0,0,1,2, -1,1,1,1,1,0,1,2,1,1,1,1,1,1,1,0,1,1,1,1,1,1,0,1,1,1,1,1,1,0,0,1, -2,3,2,3,3,2,0,1,1,1,0,0,1,0,2,0,1,1,3,1,0,0,0,0,0,0,0,1,0,0,2,1, -1,1,1,1,1,1,1,0,1,0,1,1,1,1,0,1,1,1,0,0,1,1,0,1,0,0,0,0,0,0,1,0, -2,3,3,3,3,1,2,2,2,2,0,1,1,0,2,1,1,1,2,1,0,1,1,0,0,1,0,1,0,0,2,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,2,0,0,1,1,2,2,1,0,0,2,0,1,1,3,0,0,1,0,0,0,0,0,1,0,1,2,1, -1,1,2,0,1,1,1,0,1,0,1,1,0,1,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,1,1,0, -1,3,2,3,2,1,0,0,2,2,2,0,1,0,2,0,1,1,1,0,1,0,0,0,3,0,1,1,0,0,2,1, -1,1,1,0,1,1,0,0,0,0,1,1,0,1,0,0,2,1,1,0,1,0,0,0,1,0,1,0,0,1,1,0, -3,1,2,1,1,2,2,2,2,2,2,1,2,2,1,1,0,0,0,2,2,2,0,0,0,1,2,1,0,1,0,1, -2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,2,1,1,1,0,1,0,1,1,0,1,1,1,0,0,1, -3,0,0,0,0,2,0,1,1,1,1,1,1,1,0,1,0,0,0,1,1,1,0,1,0,1,1,0,0,1,0,1, -1,1,0,0,1,0,0,0,1,0,1,1,0,0,1,0,1,0,1,0,0,0,0,1,0,0,0,1,0,0,0,1, -1,3,3,2,2,0,0,0,2,2,0,0,0,1,2,0,1,1,2,0,0,0,0,0,0,0,0,1,0,0,2,1, -0,1,1,0,0,1,1,0,0,0,1,1,0,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0, -2,3,2,3,2,0,0,0,0,1,1,0,0,0,2,0,2,0,2,0,0,0,0,0,1,0,0,1,0,0,1,1, -1,1,2,0,1,2,1,0,1,1,2,1,1,1,1,1,2,1,1,0,1,0,0,1,1,1,1,1,0,1,1,0, -1,3,2,2,2,1,0,0,2,2,1,0,1,2,2,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1, -0,0,1,1,0,1,1,0,0,1,1,0,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0, -1,0,0,1,0,2,3,1,2,2,2,2,2,2,1,1,0,0,0,1,0,1,0,2,1,1,1,0,0,0,0,1, -1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0, -2,0,2,0,0,1,0,3,2,1,2,1,2,2,0,1,0,0,0,2,1,0,0,2,1,1,1,1,0,2,0,2, -2,1,1,1,1,1,1,1,1,1,1,1,1,2,1,0,1,1,1,1,0,0,0,1,1,1,1,0,1,0,0,1, -1,2,2,2,2,1,0,0,1,0,0,0,0,0,2,0,1,1,1,1,0,0,0,0,1,0,1,2,0,0,2,0, -1,0,1,1,1,2,1,0,1,0,1,1,0,0,1,0,1,1,1,0,1,0,0,0,1,0,0,1,0,1,1,0, -2,1,2,2,2,0,3,0,1,1,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -0,0,0,1,1,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0, -1,2,2,3,2,2,0,0,1,1,2,0,1,2,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1, -0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0, -2,2,1,1,2,1,2,2,2,2,2,1,2,2,0,1,0,0,0,1,2,2,2,1,2,1,1,1,1,1,2,1, -1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,0,1,1,1,0,0,0,0,1,1,1,0,1,1,0,0,1, -1,2,2,2,2,0,1,0,2,2,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2,0, -0,0,1,0,0,1,0,0,0,0,1,0,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,2,2,2,2,0,0,0,2,2,2,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1, -0,1,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,2,2,2,2,0,0,0,0,1,0,0,1,1,2,0,0,0,0,1,0,1,0,0,1,0,0,2,0,0,0,1, -0,0,1,0,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0, -1,2,2,2,1,1,2,0,2,1,1,1,1,0,2,2,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,1, -0,0,1,0,1,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0, -1,0,2,1,2,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0, -0,0,1,0,1,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0, -1,0,0,0,0,2,0,1,2,1,0,1,1,1,0,1,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,1, -0,0,0,0,0,1,0,0,1,1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1, -2,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -1,1,1,0,1,0,1,0,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0, -1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -1,1,0,1,1,0,1,0,1,0,0,0,0,1,1,0,1,1,0,0,0,0,0,1,0,1,1,0,1,0,0,0, -0,1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0, -}; - - -const SequenceModel Koi8rModel = -{ - KOI8R_CharToOrderMap, - RussianLangModel, - (float)0.976601, - PR_FALSE, - "KOI8-R", - "Russian" -}; - -const SequenceModel Win1251Model = -{ - win1251_CharToOrderMap, - RussianLangModel, - (float)0.976601, - PR_FALSE, - "WINDOWS-1251", - "Russian" -}; - -const SequenceModel Latin5Model = -{ - latin5_CharToOrderMap, - RussianLangModel, - (float)0.976601, - PR_FALSE, - "ISO-8859-5", - "Russian" -}; - -const SequenceModel MacCyrillicModel = -{ - macCyrillic_CharToOrderMap, - RussianLangModel, - (float)0.976601, - PR_FALSE, - "MAC-CYRILLIC", - "Russian" -}; - -const SequenceModel Ibm866Model = -{ - IBM866_CharToOrderMap, - RussianLangModel, - (float)0.976601, - PR_FALSE, - "IBM866", - "Russian" -}; - -const SequenceModel Ibm855Model = -{ - IBM855_CharToOrderMap, - RussianLangModel, - (float)0.976601, - PR_FALSE, - "IBM855", - "Russian" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangCzechModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangCzechModel.cpp deleted file mode 100644 index b949c9e..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangCzechModel.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - -static const unsigned char czech_iso_8859_2CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 32, 42, 49, 41, 63, 52, 36, 38, 53, 35, 37, 59, 44, 34, 47, - 33,180, 48, 45, 40, 56, 39, 76, 70, 75, 51,255,255,255,255,255, -255, 3, 24, 19, 11, 1, 50, 57, 17, 8, 16, 12, 5, 10, 4, 2, - 14, 58, 15, 6, 7, 13, 9, 80, 62, 21, 23,255,255,255,255,255, -179,178,177,176,175,174,173,172,171,170,169,168,167,166,165,164, -163,162,161,160,159,158,157,156,155,154,153,152,151,150,149,148, -147,146,145,144,143,142,141,140,139, 64,138, 79,137,136, 61,135, -134,133,132,131,130,129,128,127,126, 28,125, 43,124,123, 25,122, -121, 71,120,119,118,117,116,115, 60, 72,114,113, 73, 69,112, 78, -111,110,109, 68,108,107,106,105, 65, 77, 67,104,103, 74,102,101, -100, 20, 99, 98, 97, 96, 95, 94, 27, 29, 93, 92, 22, 18, 91, 55, - 90, 89, 54, 66, 88, 87, 86, 85, 26, 31, 46, 84, 83, 30, 82, 81, -}; - -static const PRUint8 czechLangModel[] = -{ -1,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,1,0,0,3,3,3,3,3,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,2,0,0,0,2,2,0,2,0,0,0,0,2,0,0, -1,2,0,3,3,3,3,2,3,3,3,3,3,3,3,3,3,0,3,0,0,0,3,3,3,3,3,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,2,2,0,2,0,0,0,0,0,0,0, -1,2,1,3,3,3,3,2,3,3,3,3,2,3,3,3,3,0,3,0,0,0,3,3,3,3,3,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,2,2,0,2,0,0,0,0,0,0,0, -3,3,3,3,2,3,3,3,0,0,3,3,3,0,0,0,2,3,3,3,3,3,2,2,3,0,2,2,3,3,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,2,2,2,3,3,0,0,3,2,3,2,3,3,0,3,2,2,0,3,2,3,3,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,0,0,0, -3,3,3,3,3,2,3,3,3,3,2,3,3,3,3,2,2,3,3,3,3,1,0,2,0,0,1,0,1,2,2,0, -0,0,0,0,0,0,0,0,0,0,3,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,2,2,3,3,2,2,3,3,0,3,0,2,3,2,3,3,3,1,2,0,3,2,2,3,3,3,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,2,3,3,3,3,2,3,3,3,3,2,3,2,3,3,2,3,2,0,0,3,3,3,1,3,3,1,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,1,0,0,0,2,2,0,2,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,0,1,3,2,3,2,3,2,2,3,2,3,3,3,3,2,3,3,3,3,3,3,3,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,0,0,2,2,3,2,3,0,2,3,2,3,3,3,0,2,3,2,2,1,3,3,3,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,3,2,2,3,3,3,3,2,2,3,3,3,3,3,0,3,2,3,2,3,3,3,3,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,2,3,2,3,0,3,0,2,0,2,3,3,0,0,0,2,3,1,0,3,3,3,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,1,2,3,3,3,3,3,3,3,3,3,0,3,3,3,3,0,3,0,0,0,3,3,3,2,3,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,3,0,0,0,2,2,0,0,0,0,0,0,0,0,0, -3,3,3,2,3,3,3,3,0,0,0,2,3,2,3,2,0,3,2,3,2,3,0,0,0,3,1,2,2,2,3,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,2,3,3,3,2,3,2,3,3,2,1,3,0,3,3,3,0,3,2,3,0,2,3,3,3,3,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0, -3,2,3,3,3,3,3,3,2,3,3,2,3,2,2,0,2,3,2,3,0,0,2,2,2,1,2,3,0,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,2,3,2,3,2,2,2,3,0,3,0,0,0,3,3,3,1,0,2,2,2,1,0,3,3,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0, -0,0,0,3,3,3,3,0,3,3,3,3,0,2,3,3,3,0,3,0,0,0,3,3,3,2,3,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,3,2,3,2,0,3,3,2,2,0,3,2,2,0,1,3,3,0,1,1,0,0,0,0,0,0,0,1,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,0,0,3,3,3,3,0,3,3,3,3,0,3,3,2,3,0,3,2,0,0,3,2,3,3,3,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,3,3,3,3,0,2,2,3,3,2,3,2,2,3,0,3,0,0,0,2,3,3,2,3,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0, -0,0,0,3,3,3,3,0,3,3,3,3,0,2,2,3,3,0,3,0,0,0,2,2,3,2,3,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,2,3,3,2,3,3,0,2,3,0,3,2,2,0,2,3,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,2,2,3,2,2,2,2,3,0,3,2,0,3,2,3,3,3,1,0,0,3,0,2,2,3,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,2,3,2,2,2,3,2,0,3,3,2,0,2,0,0,3,2,3,0,0,0,2,0,0,0,2,2,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,2,3,2,2,2,2,3,1,2,0,2,2,2,0,0,0,3,1,3,0,0,0,2,0,0,2,0,0,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,3,0,3,3,0,1,0,3,2,1,1,0,0,3,2,3,0,0,0,0,0,0,0,0,0,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,3,1,3,3,2,1,0,3,2,3,2,0,0,3,2,2,0,0,0,0,0,0,2,2,2,0,1,0, -0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,0,0,3,2,2,3,0,2,3,3,2,0,2,2,2,3,0,2,0,0,0,2,2,2,2,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,2,2,3,0,3,3,2,2,0,2,2,2,2,0,3,0,0,0,2,2,2,2,2,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,1,3,3,2,0,2,3,2,2,0,1,2,3,2,0,0,0,0,0,2,2,3,2,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,0,0,3,3,2,0,0,2,2,2,0,0,2,2,0,2,0,2,1,0,0,0,2,2,0,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,2,2,2,2,0,0,0,0,2,0,3,0,0,2,0,2,0,2,0,0,0,3,0,0,0,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,0,0,0,0,2,0,0,0,0,3,0,0,0,0,0,0,3,2,2,0,0,0,0,0,0,2,0,0,2, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0, -3,2,3,0,0,2,0,2,0,2,2,0,1,0,0,0,0,2,0,3,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,2,2,2,0,2,0,0,0,3,0,2,0,3,0,0,0,0,3,0,0,0,0,0,2,0,0,0,0,0,2, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,2,2,0,0,0,0,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,0,2,2,1,2,0,0,1,0,0,0,2,0,0,2,0,2,3,2,2,0,2,0,1,3,0,2,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,0,0,0,0,2,2,1,0,0,3,0,2,0,0,2,0,0,2,2,0,0,0,2,0,0,0,0,0,0, -0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,2,2,2,0,0,2,2,0,0,0,2,0,3,0,0,2,0,2,2,2,0,0,0,2,0,0,0,0,1,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0, -2,3,2,0,1,0,0,1,0,0,0,0,3,0,2,0,0,1,0,2,3,2,0,0,0,0,0,0,0,2,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,3,0,0,0,2,0,0,0,0,0,2,0,0,3,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,2,2,0,0,3,0,0,0,0,2,0,2,0,0,2,0,2,2,2,0,0,0,0,0,0,2,0,2,1, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,2,2,1,3,2,2,2,0,2,2,2,2,0,0,0,2,2,2,2,0,2,0,0,0,0,0,0,0,0, -0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,2,3,2,0,1,2,2,2,0,3,2,2,2,0,2,0,0,0,2,0,2,3,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,2,2,2,0,2,0,2,2,0,2,0,0,2,0,2,0,0,0,2,2,0,0,2,0,0,0,0,0, -1,1,1,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,2,0,0,0,1,0,0,0,0,0,2,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,1, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0, -2,3,0,0,0,0,2,1,1,0,0,0,0,0,0,0,3,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,0,2,0,1,3,0,0,0,0,1,0,2,2,0,0,0,2,2,0,0,0,0,0,0,0,0,0,1,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,2,2,0,1,0,2,2,2,2,0,2,2,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,0,0,0,0,3,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,0,0,0,0,3,0,0,0,0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -0,2,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,0,0,1,0,0,0,0, -0,2,2,0,0,2,2,0,0,0,0,2,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,1,2,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,2,2,2,0,2,2,2,2,0,2,0,0,0,0,1,0,0,0,0,2,2,0,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,2,1,0,0,2,0,0,2,0,2,0,2,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,2,0,2,0,2,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0, -2,0,2,0,2,0,1,1,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -}; - -const SequenceModel iso_8859_2czechModel = -{ - czech_iso_8859_2CharToOrderMap, - czechLangModel, - (float)0.968107, - PR_TRUE, - "ISO-8859-2", - "czech" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangFinnishModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangFinnishModel.cpp deleted file mode 100644 index 8ff64a8..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangFinnishModel.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - -static const unsigned char finnish_WINDOWS_1252CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 30, 48, 45, 50, 22, 46, 40, 29, 27, 32, 24, 33, 21, 26, 31, - 36, 60, 34, 23, 28, 39, 35, 59, 49, 41, 54,255,255,255,255,255, -255, 1, 43, 47, 19, 5, 44, 37, 13, 2, 18, 9, 8, 12, 3, 11, - 17, 57, 15, 6, 4, 10, 14, 51, 58, 16, 56,255,255,255,255,255, -175,255,174,173,172,171,170,169,168,167,166,165,164,255,163,255, -255,162,161,160,159,158,157,156,155,154,153,152,151,255,150,149, -148,147,146,145,144,143, 63,142,141,140,139,138,137,136,135,134, -133,132,131,130,129,128,127,126,125,124,123, 25,122,121,120,119, -118,117,116,115, 38,114,113,112,111,110,109,108,107,106,105,104, -103,102,101,100, 99, 98, 42, 97, 96, 95, 94, 93, 92, 91, 90, 89, - 88, 87, 62, 86, 7, 55, 85, 52, 84, 53, 83, 82, 81, 80, 79, 78, - 77, 76, 75, 74, 73, 72, 20, 71, 70, 69, 68, 67, 61, 66, 65, 64, -}; - -static const PRUint8 finnishLangModel[] = -{ -3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,3,3,3,0,0,0,0,0,3,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,0,0,0,0,3,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,0,2,3,0,0,2,0,0,0,0,0,2,0,0,0,2,2,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,0,0,0,0,3,0,0,0,0,0,0,0, -0,0,0,0,3,0,0,0,0,0,3,3,0,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,3,3,3,3,2,3,3,3,3,3,3,3,3,2,2,0,3,0,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,0,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,0,2,2,0,0,3,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,2,2,3,0,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,0,2,2,0,0,2,0,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0, -2,3,3,3,3,3,3,3,3,2,2,3,3,3,3,3,3,3,3,2,0,0,0,0,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,2,3,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,2,2,3,3,3,2,3,3,3,2,2,2,3,3,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,0,3,3,3,3,3,3,3,3,2,3,3,3,0,0,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,0,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,3,3,3,0,0,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,0,2,2,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,3,2,3,0,0,3,3,3,0,0,2,3,3,0,0,3,0,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,0,3,3,3,3,3,3,2,3,3,3,0,3,3,3,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,0,3,2,3,0,0,3,3,0,0,0,0,3,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,2,3,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,3,0,0,0,0,0,2,2,0,0,2,0,0,0,2,0,0,0,2,2,0,0,0,0,0,0,0,0, -2,3,3,3,3,3,3,3,3,0,0,3,3,3,3,3,3,3,3,3,0,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,3,3,3,2,0,3,3,0,2,0,3,3,3,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0, -3,3,0,0,3,0,3,0,0,3,3,0,0,0,0,3,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,0,3,2,3,2,2,3,3,0,2,2,3,3,0,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,0,3,3,3,3,0,2,3,3,2,3,3,3,2,3,3,0,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,0,3,0,3,0,0,3,3,0,0,0,2,2,0,0,0,2,2,2,0,0,0,0,3,0,0,3,3,0, -0,2,0,2,0,3,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,2,3,0,3,2,2,0,3,3,0,3,0,2,0,2,0,3,3,3,2,0,3,2,3,2,2,0,0, -3,3,2,2,2,0,2,2,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,3,3,0,2,0,2,3,3,0,0,2,0,3,0,0,0,2,3,2,2,3,0,0,3,2,0,3,2,0, -2,0,0,2,0,3,2,0,3,0,0,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,0,3,0,3,3,0,3,3,0,0,0,2,3,0,0,0,2,0,2,2,3,0,0,3,0,2,3,3,0, -0,2,0,0,0,3,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,0,0,3,3,3,3,0,3,2,3,3,2,3,3, -2,2,3,3,0,2,2,2,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0, -3,3,0,0,3,0,3,0,0,3,3,0,0,0,0,3,0,0,0,0,2,3,3,2,2,2,3,3,2,3,2,2, -0,2,2,2,0,2,2,2,2,0,0,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,2,3,0,3,2,3,2,0,2,2,3,2,2,0,0,0,0,0,3,3,3,2,2,3,3,3,2,2,3,2, -3,3,2,2,0,2,2,0,2,2,2,0,2,2,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,0,3,0,3,0,0,3,3,0,0,0,2,3,2,0,0,2,2,3,2,2,2,0,3,3,2,3,2,0, -2,2,2,0,0,3,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,0,3,0,3,0,0,3,3,2,0,0,0,3,0,2,0,2,2,3,0,2,0,0,3,2,0,2,2,2, -2,2,2,0,0,3,2,0,2,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,2,2,0,3,2,2,0,2,3,2,2,0,2,3,0,0,3,2,3,2,0,3,3,2,2,3,0,2, -3,3,2,2,0,0,3,0,0,0,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,3,3,3,0,2,0,3,0,2,2,2,2,2,2,0,2,2,2,0,2,0,2,2,0,2,2,3,2,2,3,2, -2,2,2,3,0,0,2,2,0,0,0,0,2,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,0,0,2,0,3,0,0,3,3,0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,3,2,0, -0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,0,3,0,3,0,0,3,2,0,0,0,0,2,0,0,0,2,3,3,0,2,0,0,3,3,2,3,2,2, -3,0,2,2,0,2,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0, -3,3,0,0,3,0,2,0,0,3,3,0,0,0,0,2,0,0,0,2,2,3,3,2,0,2,3,2,2,3,3,2, -0,2,3,2,0,3,3,0,2,3,0,0,0,0,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0, -3,3,0,0,3,0,3,2,0,2,3,0,0,0,0,2,0,0,0,0,0,3,0,0,0,0,3,0,0,3,2,0, -0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,2,3,2,3,0,0,3,3,0,2,0,2,3,0,0,0,2,0,3,2,0,0,0,3,2,0,2,2,0, -2,2,0,2,0,2,2,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,3,2,2,2,0,2,3,2,2,0,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -0,3,0,0,0,2,2,3,2,0,0,2,2,0,0,2,0,0,0,0,2,2,2,3,0,3,3,2,2,0,0,2, -2,3,2,2,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,2,2,0,3,0,2,2,2,2,2,2,0,2,0,0,0,0,0,2,0,2,2,0,3,2,2,2,2,3,0, -3,2,2,2,0,0,2,0,0,0,0,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,2,0,0,0,0,0,2,0,0,0,3,0,0,0,0,0,0,2,0,0,0,0,2,0,0,2,2,0, -0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,2,0,3,3,0,0,2,3,0,2,0,0,0,0,3,2,0,2,2,0,2,0,3,2,2,0,0, -3,2,2,0,0,2,0,0,2,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,3,2,2,2,0,0,2, -0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,3,2,2,2,0,2,2,0,0,0,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,2,3,0,2,2,0,0,3,0,0,2,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0, -3,0,0,0,2,0,0,0,0,2,2,0,2,0,3,2,0,0,0,0,0,2,0,0,0,0,2,2,2,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,0,0,0,0,0,2,0,0,0,3,2,0,0,0,0,0,2,0,0,0,0,2,0,0,2,0,0, -0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,2,2,0,0,0,2,0,2,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0, -2,2,0,0,3,0,0,0,0,0,2,0,0,0,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,0,0,0,0,2,2,0,0,0,2,0,0,0,0,0,0,2,0,0,0,0,2,0,0,2,2,0, -0,0,0,0,0,2,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -}; - -const SequenceModel WINDOWS_1252finnishModel = -{ - finnish_WINDOWS_1252CharToOrderMap, - finnishLangModel, - (float)0.995928, - PR_TRUE, - "WINDOWS-1252", - "finnish" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangFrenchModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangFrenchModel.cpp deleted file mode 100644 index b36bb79..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangFrenchModel.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - -static const unsigned char french_WINDOWS_1252CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 30, 40, 31, 37, 27, 44, 48, 52, 32, 45, 61, 25, 29, 41, 43, - 34, 53, 38, 35, 39, 46, 42, 68, 63, 65, 71,255,255,255,255,255, -255, 2, 20, 12, 11, 1, 17, 18, 21, 4, 24, 57, 9, 13, 5, 10, - 14, 19, 7, 3, 6, 8, 16, 54, 23, 28, 36,255,255,255,255,255, - 64,180,179,178,177, 75,176,175,174,173,172,171, 91,170,169,168, -167,166, 62,111,110, 67, 78, 88,165, 97,164, 90, 82,163,162,161, -160,159,158,157,156,155,154, 86, 84, 69,101, 50,109,153, 96,152, - 81,151,108,150,100,149,148, 80,147,107,146, 51,145,144,143,142, - 70,141, 79, 72,140,139,138, 73, 77, 60, 76,137,136,135, 99,134, -106,133,132,131, 83,130, 98, 92,129, 95,128, 94,127,126,125,124, - 22, 93, 49,123,105,104, 89, 47, 26, 15, 33, 74,122,121, 56, 66, -120,103,119,118, 55,102, 87,117,116, 59,115, 58, 85,114,113,112, -}; - -static const PRUint8 frenchLangModel[] = -{ -2,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,2,3,2,0,3,2,0,0,0,3,2,0,0,0, -0,0,0,3,0,0,1,1,0,0,0,1,0,0,2,0,2,0,2,0,0,3,1,0,2,3,0,0,0,0,1,0, -2,2,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,0,2,3,0,0,0,3,0,0,0,0, -0,0,1,3,0,0,0,0,0,0,0,0,0,0,3,0,0,0,1,0,0,1,1,3,3,0,0,0,0,0,0,0, -3,3,3,3,2,3,2,3,3,3,1,3,3,3,3,1,2,2,3,2,2,0,0,1,1,3,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,2,0,0,1,0,1,2,2,0,0,0,2,0,0, -3,3,3,2,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,1,0,3,2,1,3,0,1,0,0,0,0, -1,2,0,3,0,0,0,2,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,2,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,2,3,3,3,2,2,3,3,3,3,3,2,3,0,2,2,0,2,0,2,0,1,0,0, -3,0,0,3,0,0,0,0,1,0,0,1,0,0,3,0,2,0,1,0,0,0,1,0,2,1,0,0,0,2,0,0, -3,3,3,3,1,3,3,3,2,3,1,2,2,3,3,0,1,0,0,1,3,0,0,0,0,3,0,2,0,0,0,0, -3,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1,2,0,1,0,0,1,3,1,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,0,1,1,0,3,0,3,0,0,0,2, -3,0,0,2,0,0,0,0,0,0,0,0,0,0,3,0,3,0,1,0,0,1,3,1,3,2,0,0,0,0,0,0, -3,3,3,3,3,3,3,1,3,3,3,3,3,3,3,3,3,3,3,3,2,0,3,3,0,2,0,3,0,0,0,0, -3,0,0,2,0,0,0,0,0,0,0,0,0,0,1,1,2,0,1,0,0,0,0,1,2,0,0,0,0,2,0,0, -3,3,3,3,1,3,1,3,3,3,2,3,3,3,3,2,2,3,3,3,3,3,1,0,0,3,1,3,0,0,0,0, -2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,3,0,1,0,0,0,2,1,2,2,0,0,0,3,0,0, -3,2,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,2,0,2,2,0,2,0,3,0,0,0,0, -2,0,0,3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,2,2,3,3,0,0,0,0,0, -3,3,3,3,1,2,3,3,1,3,2,1,3,0,3,2,1,2,1,2,2,0,0,2,0,3,1,2,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,1,2,2,0,2,0,0,0,2,0,0, -3,3,3,3,3,3,3,3,3,3,2,3,1,0,3,0,1,2,3,0,3,0,0,0,0,3,0,2,0,0,1,0, -1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,3,0,2,1,0,0,0,2,0,0, -3,3,2,3,3,3,1,3,2,3,0,0,3,3,3,0,0,1,0,3,0,0,0,0,0,3,0,2,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,1,0,0,0,2,1,0,2,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,1,2,0,3,3,0,0,2,0,1,3,0,2,0,0,3,0,2,0,0,0,0, -3,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,3,0,0,0,0,1,2,1,0,2,0,0,0,0,0,0, -3,3,3,2,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,2,0,0,3,0,0,0,0,0,0,0,0, -0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,0,2,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0, -3,3,0,3,0,0,3,3,0,3,0,0,0,0,3,0,0,1,0,0,0,0,0,0,0,3,0,1,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,2,0,0,0,0,0,1,0,0, -3,3,3,3,0,1,3,3,3,3,0,1,0,0,3,0,3,1,0,0,0,0,0,0,0,2,0,1,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,1,0,2,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,1,0,2,0,3,1,0,2,0,2,3,0,0,0,0,3,0,2,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,1,2,0,0,0,0,0,0,0, -0,1,1,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0, -3,3,3,3,1,2,3,3,3,3,2,1,1,0,3,1,0,0,0,3,0,0,0,2,0,2,0,3,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,0,0,2,0,1,0,0,0,0,0,0, -3,3,1,3,3,3,3,3,2,3,1,0,2,2,3,0,0,0,0,0,0,0,0,0,0,3,0,2,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,3,0,1,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,3,0,3,0,2,0,2,0,3,0,3,2,0,0,0,2,0,2,0,1,0,0,1,0,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,1,0,0,0,3,0,3,0,0,0,2,1,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0, -3,3,0,3,0,0,0,3,2,3,0,0,0,0,2,0,0,0,0,0,2,2,0,0,2,0,3,2,2,3,1,3, -0,0,1,0,1,0,2,0,1,1,2,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,2,1,2,0,0, -0,0,3,0,3,3,3,0,2,0,2,3,3,1,0,3,1,3,2,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,1,3,1,3,3,2,3,3,0,2,2,3,1,0,2,1,2,1,0,3,0,2,0,2,0,2,0,2,2,2,2, -0,2,3,1,2,3,2,2,3,1,1,1,0,2,0,1,0,0,0,2,0,0,0,0,0,0,0,0,0,0,2,0, -3,3,3,1,2,2,3,0,2,3,2,2,2,2,3,1,1,2,0,1,0,0,1,0,0,1,0,1,0,0,0,0, -0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,3,0,0,0,2,1,3,0,1,3,0,2,0,0,1,0,0,0,0,0,0,1,2,2,1,2,2,1,2, -2,2,1,0,0,0,1,1,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,1,2,3,3,2,3,3,3,0,2,2,2,3,0,2,2,2,1,3,3,0,0,2,2,0,1,2,2,1,2,2, -0,2,2,2,1,2,2,2,3,2,0,2,1,2,0,2,0,0,0,0,3,0,0,0,1,0,0,0,2,0,0,0, -3,3,0,2,0,0,2,2,3,3,0,0,0,0,2,0,1,0,0,0,3,0,0,0,2,0,2,2,0,2,1,2, -0,2,2,0,1,1,2,0,2,0,2,1,0,2,0,2,0,0,0,2,1,0,1,0,0,0,0,0,2,1,0,0, -1,0,2,0,3,2,2,0,3,1,2,2,2,0,0,1,0,1,0,1,0,0,0,0,2,0,3,0,2,2,2,3, -0,2,2,1,1,2,3,2,3,3,2,2,0,0,0,2,0,0,0,1,2,0,0,0,0,0,0,1,1,0,2,0, -0,0,0,0,2,3,0,0,2,0,0,3,3,1,0,3,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,3,0,0,3,3,3,3,0,0,0,0,2,0,0,0,0,0,3,0,0,0,2,2,2,1,0,2,1,2, -0,1,2,0,2,3,2,0,0,0,2,1,2,2,0,0,1,0,0,2,0,0,1,0,0,0,0,1,1,0,0,0, -3,3,0,3,0,2,0,3,1,3,0,2,1,2,2,0,0,0,1,0,2,0,0,0,0,1,2,2,1,2,1,2, -0,2,2,0,1,1,2,2,1,0,2,2,0,2,0,0,0,0,0,1,0,0,0,0,1,1,0,2,2,0,0,0, -3,3,0,2,1,0,1,1,1,3,0,0,1,1,2,0,0,1,0,1,1,0,0,0,0,1,0,3,0,0,0,0, -0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,3,0,0,2,3,0,2,0,0,1,2,3,0,0,0,0,0,0,0,0,1,1,2,3,0,0,2,1,2, -0,0,2,0,0,2,0,0,1,1,2,2,1,2,0,1,0,0,0,1,0,0,0,0,0,0,0,2,0,1,0,0, -3,3,0,2,0,0,0,2,0,3,0,0,0,0,3,0,0,0,0,0,2,0,0,0,2,0,3,1,2,2,2,2, -0,2,2,0,2,2,2,2,2,2,3,1,0,2,0,1,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,0, -3,3,1,2,0,0,3,3,0,3,0,1,0,0,2,0,0,0,0,0,3,0,0,0,2,1,3,1,1,2,2,3, -0,1,2,0,0,2,2,0,1,1,2,1,0,2,0,1,0,0,0,3,0,2,0,0,0,0,0,2,0,0,0,0, -3,3,0,3,0,0,3,2,3,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,2,0,1,2,1,2, -0,0,1,0,2,2,1,1,2,0,2,1,1,2,0,0,1,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0, -3,3,0,3,0,0,0,2,0,3,0,1,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,2,2, -0,2,3,0,2,1,3,1,2,1,1,2,0,2,0,2,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0, -3,3,0,3,0,0,1,2,2,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,1,0,2,0,2, -1,0,0,0,1,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,1,2,3,1,3,3,2,0,1,2,1,2,0,2,1,1,0,2,3,0,0,0,2,0,2,0,2,1,2,2, -0,2,2,0,2,2,2,2,3,2,1,2,2,2,0,1,0,0,0,0,0,0,0,0,1,0,2,0,0,0,1,0, -2,3,0,2,1,0,3,2,2,3,0,0,0,1,3,0,0,0,0,1,1,0,1,0,2,0,2,0,2,3,0,2, -1,2,2,0,0,2,0,1,1,0,2,1,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,3,0,0,0,0,0,2,0,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0, -0,0,0,0,2,0,1,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0, -0,0,0,0,3,2,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,3,0,2,2,2,2, -0,2,2,0,2,3,2,2,2,1,2,1,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,2,0, -0,3,0,0,0,0,0,3,0,3,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0, -2,3,0,2,0,0,3,3,1,3,0,0,1,0,2,0,0,0,0,0,1,0,0,0,1,0,2,1,0,2,0,1, -1,1,1,0,0,1,1,2,2,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0, -0,0,1,2,2,3,0,0,3,0,0,3,2,2,0,1,0,2,1,1,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,1,1,0,0,0,0,1,1,0,1,1,1,1,0,1,0,1,0,0,0,0,0,0,2,0,2,0,2,2,2,2, -0,2,2,0,2,1,2,2,2,2,2,1,2,1,0,1,0,0,0,2,2,0,0,0,0,0,0,1,0,0,0,0, -0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,2,0,0,0,3,0,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,3,1,0,2,0,2, -0,0,0,0,1,2,1,1,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,2,0,0,0,0, -0,1,1,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,2,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,2,3,0,0,3,0,2,0,2,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,3,3,0,0,2,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,3,0,1,1,2,1,3,0,1,2,0,1,0,1,1,0,1,2,0,0,0,0,0,0,1,0,0,0,0, -0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0, -0,0,0,0,1,3,2,0,2,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0, -0,0,0,0,0,2,1,0,2,0,2,2,1,1,0,2,0,2,0,1,0,0,0,0,2,0,2,0,2,0,2,0, -0,2,2,0,2,2,1,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,0,2,0,0,2,1,1,2,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,2,0,1,1,0,0, -0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0, -2,2,0,2,0,0,0,2,0,2,0,0,0,0,2,0,0,0,0,0,2,1,0,0,0,0,2,1,0,2,0,2, -1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0, -1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,3, -0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -}; - -const SequenceModel WINDOWS_1252frenchModel = -{ - french_WINDOWS_1252CharToOrderMap, - frenchLangModel, - (float)0.985451, - PR_TRUE, - "WINDOWS-1252", - "french" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangGermanModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangGermanModel.cpp deleted file mode 100644 index 892afb0..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangGermanModel.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - -static const unsigned char german_WINDOWS_1252CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 30, 34, 50, 25, 27, 36, 31, 28, 35, 49, 41, 39, 32, 42, 48, - 45, 58, 44, 21, 37, 40, 43, 29, 62, 61, 47,255,255,255,255,255, -255, 8, 16, 12, 9, 1, 18, 13, 7, 3, 46, 20, 11, 14, 2, 15, - 33, 53, 4, 5, 6, 10, 23, 17, 55, 51, 19,255,255,255,255,255, -180,179,178,177,176,175,174,173,172,171,170, 64,169,168,167,166, -165,164, 54,163,162,161, 66,160,159,158,157, 63,156,155,154,153, - 65,152,151,150,149,148,147,146,145,144,143, 57,142,141,140,139, -138,137,136,135,134,133,132,131,130,129,128, 56,127,126,125,124, -123,122,121,120, 59,119,118, 71,117,116,115,114,113,112,111,110, -109,108,107,106,105,104, 60,103,102,101,100, 99, 52, 98, 97, 26, - 70, 96, 69, 95, 24, 94, 68, 93, 92, 67, 91, 90, 89, 88, 87, 86, - 85, 84, 83, 82, 81, 80, 38, 79, 78, 77, 76, 75, 22, 74, 73, 72, -}; - -static const PRUint8 germanLangModel[] = -{ -3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,0,2,2,2,0,3,0,0,0,0,0,0, -3,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,3,0,2,2,2,0,2,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,0,0,0,0,0,0,0,0, -2,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,2,0,2,2,0,0,2,0,0,0,0,0,0,0, -3,3,2,3,3,3,3,2,3,2,3,3,3,3,3,3,2,3,3,3,0,0,2,2,0,3,0,0,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,0,0,0,0,0,0,0,0, -2,0,0,0,0,3,0,0,0,0,0,0,0,2,0,0,0,0,2,0,2,2,0,0,2,0,0,0,0,0,0,0, -3,2,3,3,3,3,3,3,2,3,3,3,3,2,3,2,3,2,3,2,0,2,2,3,0,0,0,0,0,0,0,0, -3,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,2,0,3,2,3,0,0,0,0,0,0,0,0, -2,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,2,3,3,2,2,3,3,3,3,2,2,3,0,3,2,3,0,0,0,0,0,0,0,0, -2,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,2,3,3,2,0,0,3,0,0,3,0,0,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,2,3,3,3,2,2,3,2,2,2,2,0,0,3,2,2,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,2,3,2,3,3,3,3,2,3,2,3,2,2,0,0,2,2,0,3,0,0,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,0,3,2,3,0,0,0,0,0,0,0,0, -2,0,0,0,0,3,0,0,0,0,0,0,0,2,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0, -2,0,2,2,0,2,3,2,0,2,2,2,0,0,2,2,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,2,3,3,0,2,2,3,2,2,2,2,3,0,2,0,2,0,0,0,0,0,0,0,0, -0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,2,3,2,3,3,2,3,3,3,3,2,3,3,3,2,2,2,2,2,0,3,0,3,0,0,0,0,0,0,0,0, -3,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -2,3,2,3,3,3,3,2,3,2,3,3,3,3,2,3,2,3,2,3,0,0,3,0,0,3,0,0,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0, -3,2,3,3,3,3,2,3,2,3,3,2,3,2,3,2,2,2,2,2,0,2,2,3,0,0,0,0,0,0,0,0, -0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,0,3,0,0,0,0,3,0,3,2,0,0,0,3,0,2,0,0,0,0,3,0,3,0,0,0,0,0,0,0,0, -0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,2,3,3,2,3,3,3,2,2,3,3,2,0,3,0,3,0,0,0,0,0,0,0,0, -2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,0,2,3,2,3,2,3,3,2,2,0,3,2,3,2,2,2,0,3,0,3,0,0,0,0,0,0,0,0, -2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,3,2,3,3,3,2,3,0,3,3,2,2,2,3,2,2,2,2,2,0,3,0,2,0,0,0,0,0,0,0,0, -0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,0,0,3,2,3,0,2,0,3,0,0,3,0,0,0,2,2,2,2,0,2,0,0,2,0,0,2,0,2, -3,0,2,0,3,2,0,0,0,0,0,0,2,0,0,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,3,0,3,3,3,3,0,2,0,3,3,3,2,0,3,0,2,0,0,0,0,0,0,0,3,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,0,0,0,0,2,0,2,0,0,0,0,3,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0, -0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,0,3,3,3,3,0,3,3,3,3,3,3,0,2,0,3,0,2,0,0,0,0,0,3,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,3,0,0,0,3,0,3,0,0,0,0,3,0,0,0,0,0,0,2,0,2,0,0,3,0,0,0,0,0, -0,0,2,0,0,2,0,0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,2,2,2,3,2,0,0,2,2,2,2,2,0,2,2,2,0,2,0,0,2,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,3,3,3,3,2,2,0,2,3,3,2,0,2,0,2,2,2,0,0,3,0,0,0,2,0,0,2,0,2,0,2, -2,2,2,0,3,0,3,0,2,3,0,3,3,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,2,0,0, -3,0,3,0,0,0,0,3,0,3,0,0,0,2,3,0,0,0,0,0,0,3,0,3,0,0,3,0,0,2,0,2, -0,0,3,0,2,3,0,0,0,2,0,0,0,0,0,2,0,0,2,2,0,0,0,0,0,0,0,2,0,0,0,0, -3,0,3,0,0,0,0,3,0,3,0,0,0,0,3,0,0,0,0,0,0,2,0,2,0,0,2,0,0,2,0,0, -0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,2,3,2,2,2,0,2,3,3,3,2,3,0,3,0,2,0,2,2,0,0,0,0,0,0,0,0,0,2,2, -2,0,0,2,2,0,2,3,0,2,0,3,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0, -3,2,2,3,0,0,0,3,0,3,3,0,0,0,3,0,0,0,0,0,0,2,0,2,0,0,2,0,0,2,0,0, -0,0,2,0,0,2,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,0,0,0,0,3,0,3,0,0,0,0,3,0,0,0,0,0,0,2,0,3,0,0,3,0,0,3,0,2, -0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0, -3,0,3,3,2,3,3,3,0,3,3,2,0,0,3,0,0,3,2,0,0,2,0,2,0,0,0,0,0,0,0,0, -3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,3,0,0,0,3,0,3,3,0,0,0,3,0,0,0,0,0,0,3,0,2,0,0,2,0,0,2,0,0, -0,0,2,0,0,2,0,2,0,0,0,2,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0, -0,3,0,2,3,2,3,0,2,0,2,3,0,3,2,2,0,0,0,0,3,0,0,0,2,0,3,0,0,0,2,2, -0,0,2,0,2,0,2,0,2,2,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,3,0,0,0,3,0,3,3,0,0,0,3,0,0,0,0,0,0,3,0,2,0,0,0,0,0,3,0,0, -0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,3,0,0,3,3,0,2,0,0,0,0,3,0,0,0,0,0,0,2,0,2,0,0,3,2,0,2,0,2, -0,0,2,0,2,2,2,2,0,0,0,2,0,0,0,3,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,3,0,3,3,3,3,0,2,0,3,3,3,2,0,2,2,3,0,0,0,0,2,0,0,3,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,0,0,0,0,3,2,3,0,0,0,0,3,0,0,0,0,0,0,2,0,2,2,0,3,0,0,2,0,0, -0,0,2,0,2,2,2,2,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,0,2,0,0,2,0,0,0,2,0,0,3,0,0,0,2,0,0,3,0,0,0,0,0,2,0,0,0,2,2, -0,0,0,0,2,0,0,0,0,2,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,0,0,0,3,0,3,3,0,0,0,3,0,0,0,0,0,0,2,0,2,0,0,2,0,0,2,0,0, -0,0,2,0,2,3,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,0,0,0,0,3,0,3,0,0,0,0,3,0,0,0,0,0,2,2,0,2,2,0,2,0,0,2,2,0, -0,2,2,0,2,0,0,0,0,2,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,0,0,0,0,3,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,2,0,0,2,0,0,2,0,0, -0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,0,0,0,2,3,0,3,0,0,0,0,2,0,0,0,0,0,2,2,0,2,0,0,3,0,0,2,3,2, -0,0,2,0,2,2,2,2,0,2,0,2,0,0,0,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,2,3,0,0,2,3,0,3,3,0,0,0,2,0,0,3,0,0,0,0,0,2,0,0,2,3,0,0,0,0, -0,0,2,0,0,2,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,0,0,0,0,0,3,0,3,0,0,0,0,2,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,0,0,0,0,2,0,3,0,0,0,0,2,0,3,0,0,0,0,2,0,2,0,0,2,0,2,0,0,0, -0,0,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,3,2,0,2,0,2,0,2,2,0,0,0,2,0,2,2,2,2,0,0,0,0,0,0,0,0,0,2,0, -2,0,0,0,0,0,2,0,2,2,0,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,0,0,0,0,0,3,0,3,0,0,0,0,2,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,0,0,3,2,0,2,2,0,0,0,2,0,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,2,2,2,2,2,2,0,2,0,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,2,2,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,0,2,2,2,0,2,0, -0,2,2,0,0,0,2,0,2,2,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,2,0,0,0,0,2,0,2,0,0,0,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -}; - -const SequenceModel WINDOWS_1252germanModel = -{ - german_WINDOWS_1252CharToOrderMap, - germanLangModel, - (float)0.985431, - PR_TRUE, - "WINDOWS-1252", - "german" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangGreekModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangGreekModel.cpp deleted file mode 100644 index 9d2043f..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangGreekModel.cpp +++ /dev/null @@ -1,230 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - -static const unsigned char greek_ISO_8859_7CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 67, 59, 45, 74, 51, 61, 69,105, 88, 99, 80, 93, 77, 95, 97, - 73,121, 92, 71, 64,110,103, 87,114,112,116,255,255,255,255,255, -255, 32, 72, 60, 43, 29, 78, 44, 58, 28, 70, 49, 41, 55, 40, 31, - 42,115, 37, 39, 38, 53, 90, 65, 98, 66,108,255,255,255,255,255, -145,140,150,180,138,144,179,155,178,154,177,176,135,175,149,174, -173,137,143,172,171,136,148,170,169,153,152,151,168,167,166,165, -164,134,107,132,127,163,162,128,147,131,161, 84,146,160,159,142, -133,126,130,129,120,141,102, 94, 89,109,118, 83, 85,139,122,125, - 91, 36, 62, 75, 57, 46,111, 82,101, 56, 54, 81, 50, 68,113, 63, - 47, 96,158, 25, 48,106, 79, 86,119,104,124,157, 17, 21, 18, 15, -123, 1, 34, 19, 23, 5, 33, 9, 27, 6, 12, 16, 13, 4, 52, 2, - 14, 7, 11, 8, 3, 10, 22, 26, 76, 24,100,117, 20, 30, 35,156, -}; - -static const unsigned char greek_WINDOWS_1253CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 67, 59, 45, 74, 51, 61, 69,105, 88, 99, 80, 93, 77, 95, 97, - 73,121, 92, 71, 64,110,103, 87,114,112,116,255,255,255,255,255, -255, 32, 72, 60, 43, 29, 78, 44, 58, 28, 70, 49, 41, 55, 40, 31, - 42,115, 37, 39, 38, 53, 90, 65, 98, 66,108,255,255,255,255,255, -127,180,179,178,177,176,175,174,173,172,171,170,169,168,167,166, -165,134,107,164,163,162,161,160,159,158,157,156,155,154,153,152, -151,137,102,132,150,149,148,128,139,131,147, 84,138,146,145,136, -133,126,130,129,120,144,143, 94, 89,109,118, 83, 85,135,122,125, - 91, 36, 62, 75, 57, 46,111, 82,101, 56, 54, 81, 50, 68,113, 63, - 47, 96,142, 25, 48,106, 79, 86,119,104,124,141, 17, 21, 18, 15, -123, 1, 34, 19, 23, 5, 33, 9, 27, 6, 12, 16, 13, 4, 52, 2, - 14, 7, 11, 8, 3, 10, 22, 26, 76, 24,100,117, 20, 30, 35,140, -}; - -static const PRUint8 greekLangModel[] = -{ -1,2,3,3,2,3,3,3,1,3,3,3,3,3,3,3,1,2,3,2,2,3,3,1,0,3,3,0,0,3,0,0, -3,3,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,3,0,1,0,0,0,0,0,0,0,0,0,0, -2,2,3,3,3,3,3,3,2,3,3,3,3,3,3,3,1,3,3,1,2,3,3,1,1,3,3,0,0,3,0,0, -2,3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,1,0,1,0,0,0,0,0,0,0,0, -3,3,2,1,3,3,3,3,3,3,1,1,2,1,3,3,3,3,1,3,3,1,0,3,0,1,1,0,0,3,1,0, -2,1,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,1,2,3,2,1,1,1,0,3,1,3,3,2,3,3,1,3,3,0,1,3,1,1,2,1,0, -1,1,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,3,1,3,3,3,1,3,3,3,3,3,3,3,2,1,3,3,2,3,3,3,0,3,2,0,0,3,0,0, -2,3,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,1,3,3,2,1,3,3,3,3,1,3,3,2,3,3,2,3,3,2,0,2,2,0,0,1,0,0, -3,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,2,2,3,3,1,3,3,2,3,2,3,3,3,3,3,2,2,3,0,3,3,0,0,2,0,0, -1,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,1,3,3,1,3,3,3,0,3,3,3,3,1,2,2,1,2,2,3,2,3,0,3,3,0,0,3,0,0, -0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,1,3,3,0,0,3,3,1,1,3,3,3,2,1,3,1,0,3,0,0,2,2,1,0,2,3,0,0,2,0,0, -1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,2,2,3,3,1,0,3,3,3,3,2,3,2,2,3,2,1,2,2,1,0,3,3,0,0,0,0,0, -3,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0, -3,3,3,2,3,3,3,2,3,3,1,2,1,2,3,3,3,3,1,3,3,2,2,2,0,1,2,0,0,2,0,0, -0,1,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,1,2,3,3,1,1,3,2,1,0,3,3,3,0,3,3,0,3,3,3,1,3,0,0,0,0,0,2,0,0, -0,3,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,3,3,3,0,3,2,1,0,0,2,3,3,3,3,0,3,3,1,0,3,0,0,0,0,0,3,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,0,3,3,3,0,3,3,3,3,0,3,0,0,3,0,0,2,3,3,0,3,2,0,0,0,0,0, -3,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,3,3,0,1,3,3,1,2,2,2,3,3,3,3,2,3,3,2,2,3,0,1,2,0,0,3,0,0, -1,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0, -1,2,3,3,2,2,3,3,1,1,3,3,3,3,0,3,0,0,3,0,0,3,3,2,0,3,3,0,0,0,0,0, -3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,3,3,0,0,3,3,0,0,3,3,3,2,0,2,0,0,3,0,0,2,2,0,0,2,3,0,0,0,0,0, -1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,1,3,3,3,3,0,3,3,0,3,3,0,3,2,3,3,3,2,3,1,2,2,0,2,0,0,0,2,0,0, -0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0, -1,2,3,3,1,2,3,3,2,1,3,2,3,3,0,3,0,0,3,0,0,2,2,1,0,2,2,0,0,0,0,0, -2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,2,1,3,3,1,1,3,3,3,3,0,3,0,0,3,0,0,3,2,2,0,3,2,0,0,0,0,0, -2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,3,3,3,1,3,2,0,0,0,0,2,2,3,3,1,2,3,1,1,3,0,0,2,0,0,2,0,0, -0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,1,1,3,3,3,1,3,3,0,0,0,0,3,0,2,2,1,3,2,1,1,2,0,0,0,0,0,3,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,1,3,3,1,1,3,3,1,1,3,2,3,3,2,2,2,2,2,1,1,2,2,0,0,2,2,0,0,0,0,0, -1,3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,0,2,2,0,0,2,3,0,2,1,2,1,1,2,1,0,1,1,1,0,2,1,2,0,0,0,2,0,0, -0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,2,0,1,0,0,0,1,0,2,1,0,0,0,0,0,1,0, -3,3,3,2,3,3,3,1,3,2,0,0,2,1,2,2,2,3,0,3,3,1,0,3,0,0,3,0,0,2,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,1,2,3,2,3,0,3,3,0,0,2,0,2,2,2,3,0,2,3,0,0,2,0,0,0,0,0,2,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,3,3, -0,0,0,0,2,3,3,3,3,3,2,3,1,0,0,0,3,0,0,0,2,0,3,0,0,1,1,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,0,2,2, -0,0,0,0,3,3,3,3,3,2,3,2,1,0,0,0,2,0,0,0,2,0,2,0,0,1,1,2,1,0,0,1, -2,3,3,3,2,1,3,3,2,0,3,2,3,3,0,3,0,0,2,0,0,2,2,2,0,2,2,0,0,0,0,0, -2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,2,2, -0,0,0,0,3,3,3,3,2,3,2,3,0,0,0,0,2,0,0,0,3,0,3,0,0,2,0,2,0,0,0,1, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2,2,0,2,1, -0,0,0,0,3,3,3,3,3,2,3,3,1,0,0,0,3,0,0,0,2,0,2,0,0,2,1,2,0,0,0,1, -2,3,0,1,3,2,0,0,2,2,0,0,1,0,2,0,2,3,0,3,2,0,0,2,0,1,0,0,0,2,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,1,0,3,2,3,1,2,2,0,0,0,0,2,3,3,2,2,2,2,0,2,2,0,0,0,0,0,2,0,1, -0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,1,3,3,1,1,3,3,1,0,3,2,3,2,0,2,0,0,2,0,0,2,2,1,0,1,2,0,0,0,0,0, -1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0, -1,0,2,3,1,2,3,2,0,3,1,2,2,3,1,2,0,0,3,0,0,2,2,0,2,2,2,0,0,1,0,0, -1,1,0,1,0,0,0,0,0,0,0,0,0,1,2,2,0,1,0,2,0,2,0,2,1,0,0,0,0,1,1,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,3,0,3,3, -0,0,0,0,2,2,2,2,2,2,2,3,1,0,0,0,2,0,0,0,2,0,2,0,0,1,1,2,0,0,0,1, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,3,0,3,3, -0,0,0,0,2,3,2,1,2,3,1,1,1,0,0,0,1,0,0,0,2,0,2,0,0,3,1,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,3,0,3,2, -0,0,0,0,2,3,2,2,2,2,1,1,1,0,0,0,2,0,1,0,2,0,2,0,0,2,1,2,1,0,0,1, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,3,0,3,3, -0,0,0,0,1,3,2,2,2,1,3,3,1,0,0,0,2,0,1,0,2,0,1,0,0,1,1,2,1,0,0,1, -0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,3,0,3,3, -0,0,0,0,1,2,2,1,2,1,2,1,0,0,0,0,2,0,0,0,2,0,2,0,0,1,0,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,0,3,2, -0,0,0,0,2,2,2,3,2,2,1,3,0,0,0,0,1,0,0,0,2,0,1,0,0,2,0,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,3,0,2,3, -0,0,0,0,2,1,3,1,1,1,2,1,0,0,0,0,1,0,0,0,2,0,1,0,0,1,0,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,0,2,2, -0,0,0,0,2,1,1,2,2,1,1,1,1,0,0,0,1,0,0,0,2,0,1,0,0,3,1,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,2,2, -0,0,0,0,2,1,1,0,1,1,1,0,1,0,0,0,1,0,3,0,1,0,1,0,0,2,1,1,3,0,0,2, -1,1,1,3,0,3,2,2,0,2,1,2,3,3,3,3,2,0,2,0,0,2,2,0,2,1,2,0,0,1,0,0, -1,1,0,1,0,0,0,0,0,0,0,0,0,1,2,1,0,1,0,2,0,1,0,2,1,0,0,0,0,1,1,0, -3,3,1,1,3,2,3,0,2,1,0,0,0,0,2,3,2,1,0,2,2,0,0,1,2,0,0,0,0,2,0,0, -0,0,2,2,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,2,0, -2,3,0,0,2,2,3,2,2,2,0,0,1,0,2,0,1,1,0,2,2,0,0,1,1,0,0,0,0,1,0,0, -2,0,2,2,0,0,0,0,0,0,0,0,0,2,1,1,0,0,0,0,0,0,0,2,0,1,0,0,0,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,2,0,2,2, -0,0,0,0,2,2,2,1,2,1,1,1,0,0,0,0,2,0,0,0,1,0,1,0,0,1,1,1,0,0,0,0, -3,2,1,1,3,2,0,0,2,2,0,0,0,3,1,0,2,1,0,2,2,0,0,2,0,0,0,0,0,1,0,0, -0,0,1,2,0,0,0,0,0,0,0,0,0,2,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,2,0, -0,0,0,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,1,0,0,1, -0,0,0,0,1,1,2,2,2,1,2,1,2,0,0,0,1,0,1,0,2,0,1,0,0,0,1,1,1,0,0,1, -3,2,1,0,3,3,0,0,2,2,0,0,0,1,2,1,2,2,1,1,2,0,1,2,0,0,0,0,0,2,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,1,2, -0,0,0,0,2,2,3,3,3,2,2,2,0,0,0,0,2,0,0,0,1,0,2,0,0,1,0,2,0,0,0,0, -3,3,1,1,2,2,2,0,1,2,0,0,0,0,1,2,2,1,0,2,1,0,0,2,0,0,0,0,0,2,0,0, -0,1,2,2,0,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,1,0,1,1,0,0,0,0,1,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,3,0,2,3, -0,0,0,0,1,1,2,1,2,2,1,1,0,0,0,0,1,0,0,0,2,0,2,0,0,1,0,1,0,0,0,0, -3,3,1,1,2,0,1,2,2,0,0,1,1,1,0,1,1,0,1,1,0,0,2,2,2,1,0,0,0,0,0,0, -0,1,1,2,0,0,0,0,0,0,0,0,0,1,1,1,0,1,0,0,0,2,0,1,1,0,0,0,0,1,2,0, -2,2,0,1,3,2,2,0,3,2,0,0,0,0,2,0,1,2,0,1,1,0,0,2,1,0,0,0,0,1,0,0, -0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,0,2,2, -0,0,0,0,2,3,1,2,1,2,2,1,0,0,0,0,1,0,0,0,2,0,1,0,0,1,0,1,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,2,3, -0,0,0,0,2,0,0,1,2,1,0,1,1,0,0,0,1,0,1,0,2,0,0,0,0,0,1,1,2,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,3,2, -0,0,0,0,2,2,2,1,2,1,1,1,1,0,0,0,2,0,0,0,2,0,1,0,0,2,1,2,1,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,3,2, -0,0,0,0,2,1,1,0,2,0,0,0,1,0,0,0,0,0,1,0,1,0,1,0,0,0,1,0,2,0,0,1, -2,3,0,0,2,3,2,0,1,1,0,0,0,0,2,2,2,1,1,2,1,0,0,0,0,0,0,0,0,1,0,0, -0,0,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0, -0,0,1,1,0,2,2,1,0,2,0,3,2,1,1,2,0,0,0,0,0,1,2,0,2,1,1,0,0,2,0,0, -0,1,0,1,0,0,0,0,0,0,0,0,0,1,1,1,0,1,0,1,0,1,0,1,1,0,0,0,0,1,1,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,3,2, -0,0,0,0,2,1,1,0,0,0,1,0,1,0,1,0,1,1,1,0,1,1,1,0,0,2,0,0,1,0,0,1, -}; - -const SequenceModel ISO_8859_7greekModel = -{ - greek_ISO_8859_7CharToOrderMap, - greekLangModel, - (float)0.944928, - PR_TRUE, - "ISO-8859-7", - "greek" -}; - -const SequenceModel WINDOWS_1253greekModel = -{ - greek_WINDOWS_1253CharToOrderMap, - greekLangModel, - (float)0.944928, - PR_TRUE, - "WINDOWS-1253", - "greek" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangHebrewModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangHebrewModel.cpp deleted file mode 100644 index 5427c74..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangHebrewModel.cpp +++ /dev/null @@ -1,220 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Simon Montagu - * Portions created by the Initial Developer are Copyright (C) 2005 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shoshannah Forbes - * Shy Shalom - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - - -/**************************************************************** -255: Control characters that usually does not exist in any text -254: Carriage/Return -253: symbol (punctuation) that does not belong to word -252: 0 - 9 - -*****************************************************************/ - -//Windows-1255 language model -//Character Mapping Table: -static const unsigned char win1255_CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, //00 -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, //10 -+253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, //20 -252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, //30 -253, 69, 91, 79, 80, 92, 89, 97, 90, 68,111,112, 82, 73, 95, 85, //40 - 78,121, 86, 71, 67,102,107, 84,114,103,115,253,253,253,253,253, //50 -253, 50, 74, 60, 61, 42, 76, 70, 64, 53,105, 93, 56, 65, 54, 49, //60 - 66,110, 51, 43, 44, 63, 81, 77, 98, 75,108,253,253,253,253,253, //70 -124,202,203,204,205, 40, 58,206,207,208,209,210,211,212,213,214, -215, 83, 52, 47, 46, 72, 32, 94,216,113,217,109,218,219,220,221, - 34,116,222,118,100,223,224,117,119,104,125,225,226, 87, 99,227, -106,122,123,228, 55,229,230,101,231,232,120,233, 48, 39, 57,234, - 30, 59, 41, 88, 33, 37, 36, 31, 29, 35,235, 62, 28,236,126,237, -238, 38, 45,239,240,241,242,243,127,244,245,246,247,248,249,250, - 9, 8, 20, 16, 3, 2, 24, 14, 22, 1, 25, 15, 4, 11, 6, 23, - 12, 19, 13, 26, 18, 27, 21, 17, 7, 10, 5,251,252,128, 96,253, -}; - -//Model Table: -//total sequences: 100% -//first 512 sequences: 98.4004% -//first 1024 sequences: 1.5981% -//rest sequences: 0.087% -//negative sequences: 0.0015% -static const PRUint8 HebrewLangModel[] = -{ -0,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,2,3,2,1,2,0,1,0,0, -3,0,3,1,0,0,1,3,2,0,1,1,2,0,2,2,2,1,1,1,1,2,1,1,1,2,0,0,2,2,0,1, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2, -1,2,1,2,1,2,0,0,2,0,0,0,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2, -1,2,1,3,1,1,0,0,2,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,0,1,2,2,1,3, -1,2,1,1,2,2,0,0,2,2,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,2,2,2,2,3,2, -1,2,1,2,2,2,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,2,3,2,2,3,2,2,2,1,2,2,2,2, -1,2,1,1,2,2,0,1,2,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,0,2,2,2,2,2, -0,2,0,2,2,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,0,2,2,2, -0,2,1,2,2,2,0,0,2,1,0,0,0,0,1,0,1,0,0,0,0,0,0,2,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,2,1,2,3,2,2,2, -1,2,1,2,2,2,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0, -3,3,3,3,3,3,3,3,3,2,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,1,0,2,0,2, -0,2,1,2,2,2,0,0,1,2,0,0,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,2,0,0,1,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,2,3,2,2,3,2,1,2,1,1,1, -0,1,1,1,1,1,3,0,1,0,0,0,0,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0, -3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,1,1,0,0,1,0,0,1,0,0,0,0, -0,0,1,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2, -0,2,0,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,3,3,3,2,3,3,3,2,1,2,3,3,2,3,3,3,3,2,3,2,1,2,0,2,1,2, -0,2,0,2,2,2,0,0,1,2,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0, -3,3,3,3,3,3,3,3,3,2,3,3,3,1,2,2,3,3,2,3,2,3,2,2,3,1,2,2,0,2,2,2, -0,2,1,2,2,2,0,0,1,2,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,1,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,2,3,3,2,2,2,3,3,3,3,1,3,2,2,2, -0,2,0,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,3,3,3,2,3,2,2,2,1,2,2,0,2,2,2,2, -0,2,0,2,2,2,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,1,3,2,3,3,2,3,3,2,2,1,2,2,2,2,2,2, -0,2,1,2,1,2,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,2,3,2,3,3,2,3,3,3,3,2,3,2,3,3,3,3,3,2,2,2,2,2,2,2,1, -0,2,0,1,2,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,3,3,3,2,1,2,3,3,3,3,3,3,3,2,3,2,3,2,1,2,3,0,2,1,2,2, -0,2,1,1,2,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,2,0, -3,3,3,3,3,3,3,3,3,2,3,3,3,3,2,1,3,1,2,2,2,1,2,3,3,1,2,1,2,2,2,2, -0,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,0,2,3,3,3,1,3,3,3,1,2,2,2,2,1,1,2,2,2,2,2,2, -0,2,0,1,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,2,3,3,3,2,2,3,3,3,2,1,2,3,2,3,2,2,2,2,1,2,1,1,1,2,2, -0,2,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0, -3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,0,1,0,0,0,0,0, -1,0,1,0,0,0,0,0,2,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,2,3,3,2,3,1,2,2,2,2,3,2,3,1,1,2,2,1,2,2,1,1,0,2,2,2,2, -0,1,0,1,2,2,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0, -3,0,0,1,1,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,2,0, -0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,1,0,1,0,1,1,0,1,1,0,0,0,1,1,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0, -0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,0,0,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0, -3,2,2,1,2,2,2,2,2,2,2,1,2,2,1,2,2,1,1,1,1,1,1,1,1,2,1,1,0,3,3,3, -0,3,0,2,2,2,2,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0, -2,2,2,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,2,2,1,2,2,2,1,1,1,2,0,1, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,2,2,2,2,2,2,2,2,1,2,2,2,2,2,2,2,2,2,2,2,0,2,2,0,0,0,0,0,0, -0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,2,1,0,2,1,0, -0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0, -0,3,1,1,2,2,2,2,2,1,2,2,2,1,1,2,2,2,2,2,2,2,1,2,2,1,0,1,1,1,1,0, -0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,1,1,1,1,2,1,1,2,1,0,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0, -0,0,2,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,1,0,0, -2,1,1,2,2,2,2,2,2,2,2,2,2,2,1,2,2,2,2,2,1,2,1,2,1,1,1,1,0,0,0,0, -0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,2,1,2,2,2,2,2,2,2,2,2,2,1,2,1,2,1,1,2,1,1,1,2,1,2,1,2,0,1,0,1, -0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,3,1,2,2,2,1,2,2,2,2,2,2,2,2,1,2,1,1,1,1,1,1,2,1,2,1,1,0,1,0,1, -0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,1,2,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2, -0,2,0,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0, -3,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,1,1,1,1,1,1,1,0,1,1,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,1,1,1,0,1,0,0,0,1,1,0,1,1,0,0,0,0,0,1,1,0,0, -0,1,1,1,2,1,2,2,2,0,2,0,2,0,1,1,2,1,1,1,1,2,1,0,1,1,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,0,1,0,0,0,0,0,1,0,1,2,2,0,1,0,0,1,1,2,2,1,2,0,2,0,0,0,1,2,0,1, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,2,1,2,0,2,0,0,1,1,1,1,1,1,0,1,0,0,0,1,0,0,1, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,1,0,0,0,0,0,1,0,2,1,1,0,1,0,0,1,1,1,2,2,0,0,1,0,0,0,1,0,0,1, -1,1,2,1,0,1,1,1,0,1,0,1,1,1,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,2,2,1, -0,2,0,1,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,1,0,0,1,0,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,1,1,1,1,1,1,1,1,2,1,0,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,1,1,0,1,1,0,1,0,0,0,1,1,0,1, -2,0,1,0,1,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,1,0,1,1,1,0,1,0,0,1,1,2,1,1,2,0,1,0,0,0,1,1,0,1, -1,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,1,0,1,1,2,0,1,0,0,0,0,2,1,1,2,0,2,0,0,0,1,1,0,1, -1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,1,0,2,1,1,0,1,0,0,2,2,1,2,1,1,0,1,0,0,0,1,1,0,1, -2,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,1,2,2,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,1,0,1, -1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,1,2,2,0,0,0,0,2,1,1,1,0,2,1,1,0,0,0,2,1,0,1, -1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,1,0,1,1,2,0,1,0,0,1,1,0,2,1,1,0,1,0,0,0,1,1,0,1, -2,2,1,1,1,0,1,1,0,1,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,1,0,2,1,1,0,1,0,0,1,1,0,1,2,1,0,2,0,0,0,1,1,0,1, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0, -0,1,0,0,2,0,2,1,1,0,1,0,1,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,1,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,1,0,1,1,2,0,1,0,0,1,1,1,0,1,0,0,1,0,0,0,1,0,0,1, -1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,2,1,1,1,1,1,0,1,0,0,0,0,1,0,1, -0,1,1,1,2,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,1,2,1,0,0,0,0,0,1,1,1,1,1,0,1,0,0,0,1,1,0,0, -}; - -const SequenceModel Win1255Model = -{ - win1255_CharToOrderMap, - HebrewLangModel, - (float)0.984004, - PR_FALSE, - "WINDOWS-1255", - "Hebrew" -}; - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangHungarianModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangHungarianModel.cpp deleted file mode 100644 index 7c221f6..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangHungarianModel.cpp +++ /dev/null @@ -1,244 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" -/**************************************************************** -255: Control characters that usually does not exist in any text -254: Carriage/Return -253: symbol (punctuation) that does not belong to word -252: 0 - 9 - -*****************************************************************/ - -//Character Mapping Table: -static const unsigned char Latin2_HungarianCharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 30, 47, 46, 43, 34, 50, 52, 38, 41, 49, 36, 44, 33, 35, 53, - 51,180, 45, 37, 42, 56, 39, 62, 64, 59, 54,255,255,255,255,255, -255, 2, 18, 27, 17, 1, 24, 12, 20, 9, 23, 7, 5, 13, 4, 8, - 26, 65, 10, 6, 3, 21, 19, 67, 63, 16, 11,255,255,255,255,255, -179,178,177,176,175,174,173,172,171,170,169,168,167,166,165,164, -163,162,161,160,159,158,157,156,155,154,153,152,151,150,149,148, -147,146,145,144,143,142,141,140,139,138,137,136,135,134,133,132, -131,130,129,128,127,126,125,124,123,122,121,120,119,118,117,116, -115, 55,114,113,112,111,110,109,108, 48,107,106,105, 66,104,103, -102,101,100, 60, 99, 61, 57, 98, 97, 96, 58, 68, 69, 95, 94, 93, - 92, 14, 91, 90, 89, 88, 87, 86, 85, 15, 84, 70, 83, 31, 82, 81, - 80, 79, 78, 25, 77, 28, 22, 76, 75, 74, 32, 40, 29, 73, 72, 71, -}; - -static const unsigned char win1250HungarianCharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 30, 47, 46, 43, 34, 50, 52, 38, 41, 49, 36, 44, 33, 35, 53, - 51,180, 45, 37, 42, 56, 39, 63, 65, 60, 54,255,255,255,255,255, -255, 2, 18, 27, 17, 1, 24, 12, 20, 9, 23, 7, 5, 13, 4, 8, - 26, 67, 10, 6, 3, 21, 19, 70, 64, 16, 11,255,255,255,255,255, -179,178,177,176, 59,175,174,173,172,171,170,169,168,167,166,165, -164,163,162,161,160,159,158,157,156,155,154,153,152,151,150,149, -148,147,146,145,144,143,142,141,140,139,138, 69,137,136,135,134, -133,132,131,130,129,128,127,126,125,124,123, 66,122,121,120,119, -118, 55,117,116,115,114,113,112,111, 48,110,109,108, 68,107,106, -105,104,103, 61,102, 62, 57,101,100, 99, 58, 71, 72, 98, 97, 96, - 95, 14, 94, 93, 92, 91, 90, 89, 88, 15, 87, 73, 86, 31, 85, 84, - 83, 82, 81, 25, 80, 28, 22, 79, 78, 77, 32, 40, 29, 76, 75, 74, -}; - -//Model Table: -//total sequences: 100% -//first 512 sequences: 94.7368% -//first 1024 sequences:5.2623% -//rest sequences: 0.8894% -//negative sequences: 0.0009% -static const PRUint8 HungarianLangModel[] = -{ -2,2,3,3,3,3,3,2,3,3,3,3,3,3,2,2,3,3,3,3,2,0,3,3,0,3,3,0,2,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2, -3,2,3,3,3,3,3,0,3,3,3,3,3,2,2,3,3,3,3,3,3,0,3,3,2,3,3,0,2,0,0,2, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,2,2,2,3,3,3,0,3,3,3,3,3,3,2,3,2,3,3,3,0,2,2, -0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,3,3,3,2,2,3,2,3,3,3,3,3,2,3,3,3,3,2,2,2,3,3,3,0,2,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,2,0,3,2, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,3,3,3,3,3,2,3,3,3,2,2,3,2,3,3,3,0,2,3,2,2,3,3,0,3,3, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,2,0,2,3,3,2,2,3,2,3,3,3,2,2,3,2,0,2,3,0,2,2, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,3,3,3,3,0,2,3,3,3,3,0,2,2,3,3,3,3,2,0,2,2,0,3,3,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,2,2,3,3,3,3,3,3,0,3,3,3,3,3,2,2,3,3,3,3,0,2,0,0,2, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,3,3,3,2,3,2,3,3,3,0,2,2, -0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,2,3,3,2,3,3,2,3,3,3,2,3,3,2,2,3,2,0,3,3,0,3,2, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,2,3,2,3,3,3,2,3,3,3,3,3,3,2,3,2,2,2,2,0,2,2, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,3,3,2,3,3,2,2,0,3,3,3,0,2,3,2,2,3,3,2,2,2,3,2,0,2,0,3,2, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,3,3,3,3,3,0,3,3,3,3,3,0,2,0,3,3,3,3,2,0,3,2,0,2,3,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,3,3,3,3,3,0,2,3,3,3,3,0,2,0,3,3,3,3,0,0,3,2,0,3,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,3,3,3,3,2,2,3,3,3,0,0,3,3,3,3,3,2,2,3,2,2,2,3,0,2,3, -0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,2,3,3,3,2,2,2,3,3,2,3,2,3,2,3,3,3,2,3,0,0,3,2,0,3,2, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,3,2,2,3,3,3,0,0,0,3,3,0,2,3,2,2,3,3,2,0,3,0,0,3,2,0,2,2, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,0,2,0,3,3,2,0,0,0,3,3,0,0,2,2,0,2,3,2,0,2,0,0,2,2,0,2,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,2,2,2,0,3,3,0,0,0,0,3,3,0,0,0,0,2,3,3,0,0,3,0,0,2,2,0,3,3, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,3,3,3,3,0,2,3,3,3,3,2,0,0,3,2,2,2,0,0,3,2,0,2,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,3,3,3,3,3,0,0,3,3,3,3,0,0,0,2,3,3,2,0,0,2,2,0,2,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,2,3,2,3,2,2,2,0,2,3,3,0,3,2,2,2,3,3,3,2,3,0,2,0,2,0,0,2, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,2,0,0,3,3,2,0,0,2,3,3,2,0,0,0,0,3,3,3,2,2,0,0,3,3,0,0,2, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,3,3,3,3,0,2,3,3,2,2,0,2,0,3,3,3,2,0,0,2,2,0,2,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,2,2,2,3,3,3,2,0,0,3,3,0,2,2,2,2,3,2,3,2,2,3,2,2,2,0,2,2, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,0,2,3,3,3,3,2,2,0,0,3,2,2,0,2,2,3,2,0,0,0,0,0,2,0,0,0,2,2, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,3,3,3,3,0,2,3,2,2,2,2,0,0,3,3,2,2,0,0,2,2,0,2,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,3,3,3,3,3,0,0,3,2,3,0,0,0,0,2,2,2,2,0,0,2,0,0,2,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,2,2,2,0,3,0,0,2,3,0,3,0,0,0,2,2,0,2,2,0,0,2,0,2,0,0,0,0,0,0, -0,2,2,2,2,0,2,0,2,2,0,2,2,0,0,0,0,0,2,0,0,2,0,2,0,0,0,0,0,0,0,0, -0,0,3,3,2,2,2,0,0,3,3,3,2,0,0,0,2,2,3,0,0,0,2,0,0,2,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,3,2,3,3,2,0,0,3,3,3,0,0,0,0,2,0,2,0,0,0,3,2,0,0,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,0,0,0,0,3,3,0,0,0,0,3,2,0,0,0,0,0,2,2,0,0,0,0,0,0,2,0,2,0, -0,2,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,3,2,2,0,0,2,3,3,2,0,0,0,2,2,0,2,2,0,2,0,0,0,0,0,0,0,0,0, -2,0,2,2,2,0,0,0,0,2,2,0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,0,0,0,0,0,3,2,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -0,2,0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,2,0,0,0,0, -3,2,0,0,0,0,0,2,3,2,0,0,0,3,3,0,0,0,0,0,2,2,0,0,0,0,0,0,2,0,2,0, -0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,2,0,2,0,0,0,2,0,0,0, -2,2,2,0,0,0,0,2,2,0,3,0,0,2,0,2,0,0,2,2,2,2,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,2,2,2,0,0,0,0,0,0,0,0, -2,3,0,0,0,0,0,3,2,0,0,0,0,3,2,0,0,0,0,0,2,0,2,0,0,0,0,0,0,2,0,0, -0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,0,0,0,0,0,2,3,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,2,2,2,2,2,0,2,2,2,0,0,0,0,0,0,2,2,0,0,0,2,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,2,2,3,0,0,0,2,2,3,0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0, -0,0,2,0,2,2,2,0,2,0,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,0,0,0,0,0,2,2,2,0,0,0,2,2,0,0,0,0,2,2,0,0,0,0,0,0,0,0,2,0,0, -0,2,0,0,2,0,0,0,2,0,0,0,0,0,0,2,0,2,0,0,0,0,2,0,2,0,0,0,0,0,0,0, -3,2,0,0,0,0,0,2,0,2,0,0,0,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,2,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0, -3,2,0,0,0,0,0,3,2,0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,2,0,0, -0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0, -2,3,0,0,0,0,0,2,2,0,0,0,0,2,2,0,0,0,0,2,2,2,0,0,2,0,0,0,0,2,0,0, -0,2,0,2,2,0,0,0,2,2,2,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0, -0,3,0,0,0,3,0,2,3,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,2,0,0,2,0,0,2,2,2,0,0,0,2,2,0,0,0,0,0,2,0,0,0,0,0,0,2,0,2,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0, -0,0,0,3,2,3,0,0,0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0, -0,0,2,0,2,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,3,0,0,0,3,0,0,0,0,2,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,0,0,0,0,0,2,0,2,0,0,0,0,2,0,0,0,0,0,2,2,0,0,0,0,0,0,2,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,0,0,2,2,0,0,0,0,2,2,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,0,0,2,2,2,0,0,0,0,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0, -0,0,2,0,3,0,2,0,0,2,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0, -0,0,2,0,2,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,0,0,0,2,0,0,2,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,2,0,0,0,0,2,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -0,0,2,0,0,0,2,0,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -0,2,0,0,0,0,2,0,2,0,0,0,0,0,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -}; - -const SequenceModel Latin2HungarianModel = -{ - Latin2_HungarianCharToOrderMap, - HungarianLangModel, - (float)0.965178, - PR_TRUE, - "ISO-8859-2", - "hungarian" -}; - -const SequenceModel Win1250HungarianModel = -{ - win1250HungarianCharToOrderMap, - HungarianLangModel, - (float)0.965178, - PR_TRUE, - "WINDOWS-1250", - "hungarian" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangPolishModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangPolishModel.cpp deleted file mode 100644 index d247fa8..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangPolishModel.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - -static const unsigned char polish_iso_8859_2CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 34, 39, 37, 46, 54, 59, 51, 43, 35, 42, 45, 49, 41, 32, 47, - 31,180, 50, 33, 38, 53, 60, 30, 61, 58, 40,255,255,255,255,255, -255, 1, 21, 11, 13, 3, 44, 23, 26, 2, 18, 15, 20, 14, 6, 4, - 16, 63, 7, 8, 12, 19, 52, 9, 64, 10, 5,255,255,255,255,255, -179,178,177,176,175,174,173,172,171,170,169,168,167,166,165,164, -163,162,161,160,159,158,157,156,155,154,153,152,151,150,149,148, -147, 70,146, 57,145,144, 55, 78,143,142,141,140, 74,139,138, 56, -137, 24,136, 17,135, 82, 27,134,133,132,131, 65, 48,130,129, 25, -128,127,126,125,124,123, 77,122,121,120, 66,119,118,117,116,115, -114, 68,113, 62,112,111,110,109,108,107,106,105, 81,104,103,102, -101, 76, 69,100, 80, 99, 29, 71, 98, 67, 22, 97, 96, 95, 79, 73, - 94, 36, 93, 28, 92, 91, 75, 90, 89, 88, 87, 86, 72, 85, 84, 83, -}; - -static const PRUint8 polishLangModel[] = -{ -2,2,2,2,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,0,3,0,3,2,3,0,3,0,0,0, -0,0,0,3,0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,3,3,3,3,3,0,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,0,3,3,3,0,0,0, -0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0, -2,2,2,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,3,3,0,3,0,3,2,3,0,3,0,0,0, -0,0,0,3,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,2,2,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,3,3,0,3,0,3,2,3,0,3,0,0,0, -0,0,0,3,0,0,0,0,0,0,0,3,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,3,3,0,3,3,3,3,0,3,2,0,2,3,0,0,3,3,3,0,0,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,2,3,3,2,3,3,3,2,2,2,3,3,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,2,3,3,3,3,0,3,3,3,3,2,3,2,0,2,0,3,0,2,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,0,3,3,3,3,2,3,3,3,2,3,2,2,3,0,3,0,0,3,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,3,3,3,3,3,3,0,3,3,3,3,3,3,3,3,2,3,3,2,3,2,3,2,3,0,3,0,0,0, -0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,0,0,2,3,0,2,0,2,3,0,0,2,3,2,0,3,0,3,0,3,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,0,3,3,3,3,3,3,2,2,2,3,2,3,2,3,3,0,3,0,3,0,2,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,2,3,3,3,3,2,3,2,2,2,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,0,3,3,2,2,3,2,2,2,2,3,3,2,2,3,2,3,3,2,3,2,0,2,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,0,3,3,3,3,0,2,3,0,0,2,0,3,0,3,3,2,3,2,3,3,0,2,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,0,3,3,3,0,3,2,2,0,0,2,2,3,0,3,3,0,3,0,2,0,2,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,3,2,3,0,2,0,3,2,3,2,2,3,0,2,0,3,0,3,3,3,3,2,0,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,3,3,2,0,3,2,3,3,2,2,0,0,3,2,2,3,2,3,2,0,3,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,2,3,3,3,3,3,0,3,3,3,3,3,3,3,3,0,3,3,0,3,0,3,0,3,0,2,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,0,3,2,0,3,2,2,3,3,0,0,2,3,3,3,3,2,3,2,0,2,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,3,2,2,3,2,2,0,0,2,0,3,2,3,3,0,3,0,3,0,0,0,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,3,0,0,3,0,0,3,3,3,0,3,3,3,0,0,2,3,0,2,0,3,0,3,0,2,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,0,3,3,0,3,0,0,0,3,2,0,0,3,2,3,3,0,3,0,3,2,2,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,3,0,0,3,2,0,3,3,3,0,3,3,3,0,0,0,2,0,3,0,3,0,2,0,3,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,3,0,3,0,2,2,3,2,0,3,2,3,0,2,0,2,2,2,3,2,3,0,0,0,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,0,3,2,0,3,3,3,2,0,2,0,0,3,0,3,2,0,2,0,2,2,0,2,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,3,3,0,3,0,3,0,0,3,0,3,0,0,0,3,2,0,0,0,0,0,0,0,3,0,0,0, -0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,2,0,3,2,3,0,3,2,3,2,2,2,3,3,0,3,3,0,3,0,3,0,2,0,2,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,2,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,2,2,2,3,0,3,2,2,0,0,2,2,2,0,0,0,0,2,0,2,0,0,2,2,0,0,0,2, -2,2,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0, -3,3,2,3,0,0,3,2,0,2,0,0,0,0,0,0,2,0,2,2,0,0,0,0,0,0,0,2,0,0,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,2,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0, -2,3,2,2,3,2,0,0,2,2,2,3,0,2,3,3,3,0,0,0,0,0,0,2,0,2,0,0,0,0,2,0, -0,0,2,0,0,2,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,2,2,2,2,0,0,2,0,0,2,2,0,0,0,0,3,2,0,0,0,2,0,0,0,0,2,0,2, -2,0,0,0,2,0,0,2,2,2,0,0,2,2,0,0,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,2,0,2,2,0,2,0,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,0,0,2,2,0,0,0, -0,0,0,0,0,0,0,3,0,0,3,0,0,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,3,3,0,2,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,3,0,2,0,0,0,0, -0,2,2,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,0,0,3,0,2,2,2,0,0,0,0,0,2,0,2,0,0,2,0,2,0,0,0,0,0,2,0,0, -0,2,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,2,3,0,0,2,0,0,3,0,0,0,0,0,0,2,0,2,2,0,2,0,2,0,0,0,2,0,0,0,0, -0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,2,2,2,0,3,2,0,2,0,0,2,2,2,0,0,2,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0, -0,2,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0, -3,3,2,3,0,2,2,0,0,2,0,0,0,0,0,0,2,0,2,0,0,2,0,0,0,0,0,3,0,0,0,2, -0,2,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,3,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,3,0,2,2,0,0,0,0,2,0,0,2,0,0,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,0,3,0,2,3,2,2,0,0,3,0,0,0,0,2,0,2,2,0,2,0,0,0,0,0,0,0,0,0,0, -0,2,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -3,2,2,3,3,0,2,0,2,2,0,0,0,0,0,0,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,0,0,0,0,2,0,0,0,0,0,0,2,2,0,2,0,0,0,2,0,0,0,2,0,0,0,0,0,0, -0,0,0,0,0,2,0,2,2,0,2,3,3,0,2,2,0,2,0,2,2,0,2,0,0,0,2,0,0,2,2,2, -2,0,0,0,2,2,0,0,2,0,0,0,2,2,0,0,2,2,0,0,0,0,2,0,2,0,0,0,0,0,0,2, -0,0,0,0,0,3,2,0,2,0,2,0,2,2,0,0,0,0,0,2,2,0,0,0,0,0,0,0,2,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,0,0,0,0,2,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,0,0,0,0,0,0,0,0,0,0,0,2,0,2,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0, -2,0,2,3,2,0,0,0,2,2,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,2,0,2,0,0, -0,2,0,0,0,2,0,2,0,0,0,0,0,0,2,0,0,0,0,0,2,2,0,0,0,2,0,0,0,2,0,0, -2,2,2,2,0,2,3,0,2,0,0,0,3,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,2,2,0,2,0,2,2,2,2,0,2,0,0,2,0,0,0,0,0,2,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,2,2,0,0,0,0,0,0,0,0,0,2,0,2, -2,0,0,0,2,0,0,2,2,2,0,0,2,0,0,0,2,2,2,0,0,0,2,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,2,0,0,2,0,2,0,0,2,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,3,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0, -0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2, -0,0,0,0,2,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -}; - -const SequenceModel iso_8859_2polishModel = -{ - polish_iso_8859_2CharToOrderMap, - polishLangModel, - (float)0.974309, - PR_TRUE, - "ISO-8859-2", - "polish" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangSpanishModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangSpanishModel.cpp deleted file mode 100644 index dc3862f..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangSpanishModel.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - -static const unsigned char spanish_windows_1252CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 32, 44, 29, 38, 26, 48, 49, 53, 37, 55, 63, 31, 35, 40, 46, - 30, 58, 36, 33, 41, 47, 54, 52, 64, 51, 67,255,255,255,255,255, -255, 2, 15, 11, 9, 1, 19, 16, 21, 6, 27, 43, 8, 13, 4, 3, - 14, 22, 7, 5, 10, 12, 17, 42, 39, 18, 24,255,255,255,255,255, -180,179,178,177,176, 71,175,174,173,172,171,170,169,168,167,166, -165, 76, 70, 61, 62,164, 57,163,162,161,160,159,158,157,156,155, -154, 56,153,152,151,150,149,148,147, 79, 73, 59,146,145,144,143, - 83, 82,142,141,140,139,138,137,136,135, 78, 60,134,133,132, 50, -131, 72,130,129,128,127,126,125,124, 66,123,122,121, 68,120,119, -118, 74,117, 69,116,115,114,113,112,111, 81,110,109,108,107,106, -105, 25,104,103,102,101,100, 77, 75, 28, 99, 98, 97, 23, 96, 95, - 94, 34, 80, 20, 93, 92, 91, 90, 89, 88, 45, 87, 65, 86, 85, 84, -}; - -static const PRUint8 spanishLangModel[] = -{ -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,3,3,3,2,0,3,0,0,2,2,0, -2,3,0,0,0,0,3,0,2,3,2,3,2,0,0,0,0,0,0,2,0,0,0,0,2,0,0,2,0,2,0,2, -3,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,0,0,3,0,0,0,0,0, -0,3,0,0,0,0,2,0,0,2,2,0,3,0,0,0,0,0,0,0,0,2,0,0,2,0,0,2,0,2,0,0, -2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,2,2,3,0,0,3,2,0,0,0,0, -3,3,2,0,0,0,2,0,0,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,2,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,3,3,2,3,3,3,2,0,3,3,2,0,0,0, -0,0,0,0,0,2,0,0,0,0,3,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,3,3,0,3,3,3,3,3,3,3,3,3,2,2,3,3,3,3,3,0,2,0,0,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,2,0,0, -3,3,3,3,3,0,3,3,3,3,3,3,3,3,3,3,3,0,3,3,0,3,0,3,2,0,3,3,2,2,2,0, -0,3,0,0,0,0,2,0,0,0,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,0,2,3,0,0,0,2, -0,0,0,0,0,2,0,0,2,0,3,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -3,3,3,0,3,3,2,3,3,3,3,3,3,2,2,3,3,2,2,3,2,3,3,3,3,0,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -3,3,3,0,2,3,3,2,0,2,2,3,3,0,0,2,2,0,2,3,2,2,3,0,3,0,0,2,2,0,2,0, -0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,3,3,3,2,0,3,3,3,2,3,0,0,0,2,0,3,3,0,3,2,3,0,0,3,0,2,0,0, -0,0,0,0,0,0,0,0,0,3,0,2,3,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -3,3,3,3,2,3,3,3,0,3,3,3,2,0,0,0,0,2,0,3,3,0,3,0,3,0,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,3,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,3,3,2,2,0,2,3,3,3,0,3,3,0,0,0,0, -0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,0,0,0,0,0,3,3,3,3,0,0,2,0,3,0,0,3,0,3,0,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,2,3,3,3,0,3,2,3,2,2,0,0,0,3,0,3,3,0,2,2,3,0,0,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,3,3,3,3,2,2,0,3,2,0,0,0,2,2,0,2,0,0,3,0,2,0,3,2,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,3,3,0,2,0,3,2,0,0,2,0,2,0,3,3,0,3,0,2,0,0,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,0,3,2,0,0,0,0,3,0,0,0,0,0,0,0,2,0,0,3,0,2,0,0,3,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,2,0,3,2,0,2,2,2,2,2,3,0,0,0,0,3,0,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -3,3,3,0,0,3,3,3,0,3,0,3,0,0,0,0,0,0,2,2,0,0,3,0,2,0,0,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,3,3,0,3,3,3,0,2,0,3,2,0,2,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,0,3,2,2,0,3,0,3,0,0,0,0,0,2,0,2,0,0,2,0,2,0,0,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,3,0,2,2,3,3,3,0,2,2,2,2,2,0,3,0,0,2,0,0,0,0,2,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,2,0,2,0,0,0,0,2,2,3,0,0,2,0,0,0,3,0,2,0,0,2,0,0,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -0,0,0,3,3,0,3,3,0,3,3,2,3,3,2,3,2,0,3,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,3,3,0,3,3,2,2,2,3,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2, -2,0,0,3,0,2,2,3,2,0,0,2,0,0,2,2,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2, -3,3,3,0,0,3,0,0,0,0,0,3,0,0,0,0,0,0,0,3,0,0,2,0,2,0,0,2,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,3,3,0,3,3,3,3,3,0,2,2,2,2,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,0,3,3,2,0,0,0,3,0,0,0,0,0,3,0,3,3,0,0,0,2,2,0,0,0,0,2,2, -0,0,0,2,3,3,0,2,2,0,0,0,0,2,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,2,0, -3,3,3,0,2,3,3,2,0,0,0,3,0,0,0,0,0,0,0,0,2,0,0,0,2,2,0,2,2,0,3,2, -2,0,0,2,2,3,0,0,0,0,0,0,2,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,0,3,0,2,0,0,0,3,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,2,0,2,3, -0,0,2,0,3,2,0,0,0,0,0,0,0,2,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,3,2,0,3,3,3,2,3,3,3,2,2,2,2,2,0,0,2,2,0,0,0,0,0,0,2,2,2,0, -3,0,2,3,2,2,0,2,2,0,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0, -3,3,3,0,0,3,2,2,0,3,3,3,2,2,0,0,0,2,0,2,0,0,2,0,0,2,0,0,3,2,2,2, -0,0,2,0,2,2,0,0,2,0,0,2,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,3,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,0,3,2,0,0,0,2,3,0,0,0,0,0,0,0,2,0,0,0,0,2,2,0,2,0,2,2,2, -2,0,0,0,3,0,0,0,0,0,0,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,0,0,3,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,2,2, -2,0,2,2,2,0,0,2,2,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,3,2,0,2,2,0,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,2,2,2,2, -2,0,2,0,3,2,0,3,3,0,0,2,0,2,0,2,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2, -3,2,3,0,0,3,2,0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,3,0,2,0,2,2,3, -0,0,0,2,3,0,0,0,0,0,0,0,0,2,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,3,0,0,0,3,3,0,0,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,0,0,3,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,2,0,2,3, -2,0,0,2,2,2,0,0,2,0,0,0,0,2,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,0,3,3,0,0,0,0,2,0,0,0,0,0,2,0,0,3,0,0,0,0,2,0,0,0,2,0,2, -0,0,2,2,2,0,0,0,2,2,0,2,2,2,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2, -3,3,2,2,2,3,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,2,2,3,3,0,2,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,0,3,3,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,2,0,2,2,2, -0,0,2,2,2,0,0,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,3,3,0,2,3,0,2,0,0,3,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,3,2,0,2,2,0,3,2,2,0,3,2,0,0,2,0,0,2,0,0,0,0,0,0,0,3,2,2,0, -2,0,2,2,2,2,2,2,2,0,0,2,0,2,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0, -0,0,0,3,2,0,0,2,3,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0, -2,0,0,2,0,0,0,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,0,0,3,3,2,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2, -0,0,0,0,2,2,0,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,0,2,2,2,0,0,0,3,0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,2,2,0, -2,0,0,0,2,0,0,2,0,0,0,0,0,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,2,2,0,0,2,2,2,2,0,0,2,0,0,2,2,0,0,0,2,0,0,0,2,0,0,3,3,2,2, -2,0,2,0,2,2,0,2,2,0,0,0,0,0,2,0,0,0,3,0,2,2,0,0,0,3,0,0,0,0,0,0, -0,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,0,0,2,0,0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,2, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,2,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,2, -0,0,0,0,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,2,0,0,2,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,2,0,0,0,0,0,2,0,0,2,0,0,0,2,0,0,0,2,0,0,0,2,0,0,2,2,2,2, -2,0,2,0,0,2,0,0,2,0,0,0,0,2,0,0,0,0,2,0,2,2,0,0,0,2,0,0,0,0,0,0, -2,2,2,0,0,0,0,2,0,0,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,2,2,0,0,2,2,2,2,2,2,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0, -0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,0,0,2,0,0,2,2,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,2,0, -0,0,0,2,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, -0,0,2,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -}; - -const SequenceModel windows_1252spanishModel = -{ - spanish_windows_1252CharToOrderMap, - spanishLangModel, - (float)0.983906, - PR_TRUE, - "WINDOWS-1252", - "spanish" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangSwedishModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangSwedishModel.cpp deleted file mode 100644 index ad7e1d9..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangSwedishModel.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - -static const unsigned char swedish_WINDOWS_1252CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 31, 44, 47, 26, 33, 36, 38, 27, 40, 30, 42, 43, 29, 34, 35, - 45, 70, 46, 28, 37, 48, 32, 53, 61, 60, 57,255,255,255,255,255, -255, 1, 23, 21, 7, 2, 18, 11, 14, 9, 24, 13, 8, 12, 3, 10, - 22, 52, 5, 6, 4, 19, 16, 55, 41, 25, 54,255,255,255,255,255, -180,179,178,177, 66,176, 71,175,174,173,172,171,170,169,168,167, -166,165, 56, 59,164,163,162,161,160,159,158,157,156,155,154,153, -152,151,150, 69,149,148,147,146,145,144,143,142,141,140,139,138, - 72, 85, 77, 62,137,136,135,134,133,132,131, 39, 73, 68, 76,130, -129,128,127,126, 49, 50, 84,125,124, 81,123,122,121,120,119,118, -117,116,115,114,113,112, 58,111,110,109,108,107, 83,106,105,104, - 64,103, 80,102, 15, 17, 65, 74, 67, 51, 79, 78,101,100, 99, 98, - 97, 96, 95, 94, 82, 93, 20, 92, 91, 90, 89, 75, 63, 88, 87, 86, -}; - -static const PRUint8 swedishLangModel[] = -{ -2,2,3,3,3,3,3,3,2,2,3,3,3,2,1,3,2,3,3,0,3,3,3,2,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,3,0,0,0,0,0,0,0,0,0,0,0,0,2,1,2,0,0,2,0,0,0,0,0, -3,3,3,3,3,3,3,3,2,2,3,3,3,3,2,3,2,3,3,1,3,3,3,3,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,3,0,0,0,0,0,0,0,0,0,0,2,0,2,2,2,0,0,1,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,1,0,0,0,0,0,0,0,0,0,2,1,0,2,0,2,0,0,2,0,0,0,0,0, -3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,3,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,2,1,0,0,2,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,2,1,0,0,0,0,0,0,1,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,2,2,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,2,3,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,1,1,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,2,3,3,3,3,3,1,3,0,3,3,2,3,3,3,1,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,3,3,3,3,2,2,3,3,3,3,2,3,2,3,3,2,3,3,3,3,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,1,0,3,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,0,2,3,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,2,1,0,0,0,0,0,0,0,0,0,2,0,0, -3,3,3,3,3,3,2,3,3,3,3,2,2,3,3,3,3,2,3,3,0,2,2,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,2,2,0,2,3,3,1,2,0,1,3,3,3,0,3,3,0,0,0,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,0,0,0,0,0,0, -1,2,3,3,3,3,3,3,0,1,3,3,3,2,2,3,0,3,0,0,3,3,2,2,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,2,2,2,2,3,2,3,2,3,2,1,2,3,2,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,3,3,3,3,3,3,0,1,3,2,3,2,2,3,2,2,0,1,0,2,2,2,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,3,3,3,2,3,3,3,2,2,2,2,3,3,3,3,3,3,0,2,2,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1,0, -2,2,3,3,3,3,3,3,2,2,3,3,3,2,0,3,2,3,2,0,3,3,3,1,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,3,3,3,3,3,3,2,1,3,3,3,2,0,3,0,3,1,1,2,3,2,3,1,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0, -2,3,0,2,2,2,0,2,3,2,0,0,3,3,0,0,0,0,2,0,2,0,0,0,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,2,3,3,3,2,3,2,3,3,2,3,3,3,2,0,3,2,2,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0, -3,3,2,2,3,2,2,3,3,3,0,0,0,1,3,0,3,0,3,3,1,0,3,3,3,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0, -3,3,2,2,0,2,3,3,1,3,0,1,3,0,3,0,2,2,3,3,0,0,1,0,2,0,0,0,0,0,0,0, -0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,3,3,3,3,3,0,2,3,3,3,2,0,2,2,3,1,0,3,3,2,2,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,0,2,0,0,0,2,2,0,0,0,0,3,1,3,0,3,2,0,0,0,2,2,0,0,0,0,1,2,0, -2,0,1,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,0,0,0,0,0,2,3,0,2,0,0,3,3,2,0,3,3,0,0,0,2,2,0,0,0,0,0,2,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,3,0,1,0,2,3,3,0,2,3,0,2,3,3,0,2,2,3,2,0,2,2,0,0,2,0,2,2,0, -2,0,2,1,2,0,0,2,0,2,1,1,2,0,0,0,0,1,2,0,0,0,2,0,0,2,0,0,0,0,1,0, -3,3,0,0,0,0,0,0,3,3,0,0,0,0,3,0,3,0,2,2,0,0,0,2,2,0,0,0,0,0,2,0, -1,0,2,0,2,0,0,1,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,0,0,0,0,0,0,0,3,0,0,0,0,2,0,0,2,3,2,0,0,0,0,0,0,0,0,0,0,0,0, -1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0, -0,1,3,2,2,3,2,3,0,0,1,2,2,2,0,2,0,2,2,0,2,2,2,0,0,2,0,2,1,0,0,1, -0,2,0,1,2,2,0,0,2,0,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,0,2,0,0,0,3,2,0,0,0,0,3,0,2,0,1,1,0,0,0,0,0,0,0,0,0,0,2,0, -2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,3,3,3,2,2,3,1,0,2,2,2,2,0,2,0,2,2,0,2,1,0,2,2,2,1,1,2,0,0,0, -0,2,0,0,2,1,0,0,2,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0, -2,3,0,0,0,0,0,0,3,3,0,0,0,0,3,0,3,0,3,2,0,0,0,0,2,2,0,2,0,0,2,0, -2,2,2,0,1,2,0,2,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,1,2,1,2,2,2,2,0,2,0,3,2,0,0,2,0,2,2,0,3,2,2,1,0,1,0,0,2,0,1,0, -0,2,0,1,2,0,0,0,2,1,1,0,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,0,0,3,0,0,2,3,3,0,0,0,0,2,0,2,0,2,3,0,0,0,2,2,0,0,0,0,2,1,0, -2,0,2,1,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0, -3,2,0,0,3,1,0,0,3,3,0,0,0,2,2,2,2,0,2,2,0,1,0,2,3,0,0,1,1,2,2,0, -2,2,2,0,2,0,0,2,0,0,2,0,0,2,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,1,0,3,0,0,2,2,2,0,0,0,0,2,0,2,0,3,2,0,0,0,1,0,0,0,1,0,0,2,0, -2,0,2,0,0,0,0,0,0,0,1,0,0,2,0,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,2,1,2,2,2,1,1,2,2,2,2,0,2,0,2,1,0,0,2,2,1,0,2,2,2,2,2,2,2, -2,2,2,2,2,2,0,2,0,2,2,2,2,1,0,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,3,2,1,2,2,2,0,0,1,2,0,0,0,1,0,2,0,0,2,0,2,0,0,1,0,2,2,0,0,2, -1,2,2,0,2,2,0,2,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0, -3,3,2,3,0,2,1,2,3,2,0,0,2,2,0,0,0,2,1,0,1,2,2,0,1,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,2,0,2,0,0,3,2,3,0,0,0,0,2,2,1,0,2,2,0,0,0,1,2,0,1,0,0,0,2,0, -2,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,2,0,0,0,0,0,0,3,2,0,0,0,0,2,0,2,0,3,2,0,0,0,2,2,0,0,2,0,0,2,0, -2,2,1,1,0,0,0,2,0,0,2,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0, -3,3,0,0,3,0,1,3,2,2,0,0,0,0,2,0,2,0,2,2,0,0,0,2,2,0,0,0,0,0,0,0, -2,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,0,1,2,0,0,2,2,2,0,0,0,1,1,0,3,1,2,0,0,0,0,0,0,0,0,0,0,0,2,0, -1,0,2,0,0,0,0,2,0,0,0,0,2,1,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,0,0,0,2,2,0,0,0,2,2,0,2,0,2,2,0,0,0,0,2,2,0,2,1,0,2,0, -2,2,2,0,2,2,0,2,0,1,2,1,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0, -3,2,0,0,2,0,0,2,2,2,0,0,0,3,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0, -0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,3,3,2,2,1,3,0,0,1,2,0,0,0,0,0,0,0,0,0,2,0,2,0,1,0,2,0,0,0,0, -0,2,0,1,2,2,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,2,3,0,2,2,0,0,1,1,0,2,0,2,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0, -0,1,0,0,2,1,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,2,2,2,0,1,0,0,0,0,2,2,0,2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0, -0,2,0,0,2,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,2,2,1,2,2,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,0,0,0,1,0,0,2,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0, -2,2,2,0,0,1,0,1,2,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,1,2,2,0,0, -0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,2,0,2,0,0,0,0,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,1,0,2,2,2,0,0,0,2,0,0,0,0,2,0,2,0,0,0,2,0,0,0,1,0,0,0,0,0,1, -0,0,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,2,2,0,0,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1, -0,0,0,0,0,0,0,2,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -}; - -const SequenceModel WINDOWS_1252swedishModel = -{ - swedish_WINDOWS_1252CharToOrderMap, - swedishLangModel, - (float)0.989923, - PR_TRUE, - "WINDOWS-1252", - "swedish" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangThaiModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangThaiModel.cpp deleted file mode 100644 index 1e4f8b4..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangThaiModel.cpp +++ /dev/null @@ -1,222 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - - -/**************************************************************** -255: Control characters that usually does not exist in any text -254: Carriage/Return -253: symbol (punctuation) that does not belong to word -252: 0 - 9 - -*****************************************************************/ - -//The following result for thai was collected from a limited sample (1M). - -//Character Mapping Table: -static const unsigned char TIS620CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, //00 -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, //10 -+253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, //20 -252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, //30 -253,182,106,107,100,183,184,185,101, 94,186,187,108,109,110,111, //40 -188,189,190, 89, 95,112,113,191,192,193,194,253,253,253,253,253, //50 -253, 64, 72, 73,114, 74,115,116,102, 81,201,117, 90,103, 78, 82, //60 - 96,202, 91, 79, 84,104,105, 97, 98, 92,203,253,253,253,253,253, //70 -209,210,211,212,213, 88,214,215,216,217,218,219,220,118,221,222, -223,224, 99, 85, 83,225,226,227,228,229,230,231,232,233,234,235, -236, 5, 30,237, 24,238, 75, 8, 26, 52, 34, 51,119, 47, 58, 57, - 49, 53, 55, 43, 20, 19, 44, 14, 48, 3, 17, 25, 39, 62, 31, 54, - 45, 9, 16, 2, 61, 15,239, 12, 42, 46, 18, 21, 76, 4, 66, 63, - 22, 10, 1, 36, 23, 13, 40, 27, 32, 35, 86,240,241,242,243,244, - 11, 28, 41, 29, 33,245, 50, 37, 6, 7, 67, 77, 38, 93,246,247, - 68, 56, 59, 65, 69, 60, 70, 80, 71, 87,248,249,250,251,252,253, -}; - - - - -//Model Table: -//total sequences: 100% -//first 512 sequences: 92.6386% -//first 1024 sequences:7.3177% -//rest sequences: 1.0230% -//negative sequences: 0.0436% -static const PRUint8 ThaiLangModel[] = -{ -0,1,3,3,3,3,0,0,3,3,0,3,3,0,3,3,3,3,3,3,3,3,0,0,3,3,3,0,3,3,3,3, -0,3,3,0,0,0,1,3,0,3,3,2,3,3,0,1,2,3,3,3,3,0,2,0,2,0,0,3,2,1,2,2, -3,0,3,3,2,3,0,0,3,3,0,3,3,0,3,3,3,3,3,3,3,3,3,0,3,2,3,0,2,2,2,3, -0,2,3,0,0,0,0,1,0,1,2,3,1,1,3,2,2,0,1,1,0,0,1,0,0,0,0,0,0,0,1,1, -3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,3,3,2,3,2,3,3,2,2,2, -3,1,2,3,0,3,3,2,2,1,2,3,3,1,2,0,1,3,0,1,0,0,1,0,0,0,0,0,0,0,1,1, -3,3,2,2,3,3,3,3,1,2,3,3,3,3,3,2,2,2,2,3,3,2,2,3,3,2,2,3,2,3,2,2, -3,3,1,2,3,1,2,2,3,3,1,0,2,1,0,0,3,1,2,1,0,0,1,0,0,0,0,0,0,1,0,1, -3,3,3,3,3,3,2,2,3,3,3,3,2,3,2,2,3,3,2,2,3,2,2,2,2,1,1,3,1,2,1,1, -3,2,1,0,2,1,0,1,0,1,1,0,1,1,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0, -3,3,3,2,3,2,3,3,2,2,3,2,3,3,2,3,1,1,2,3,2,2,2,3,2,2,2,2,2,1,2,1, -2,2,1,1,3,3,2,1,0,1,2,2,0,1,3,0,0,0,1,1,0,0,0,0,0,2,3,0,0,2,1,1, -3,3,2,3,3,2,0,0,3,3,0,3,3,0,2,2,3,1,2,2,1,1,1,0,2,2,2,0,2,2,1,1, -0,2,1,0,2,0,0,2,0,1,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0, -3,3,2,3,3,2,0,0,3,3,0,2,3,0,2,1,2,2,2,2,1,2,0,0,2,2,2,0,2,2,1,1, -0,2,1,0,2,0,0,2,0,1,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0, -3,3,2,3,2,3,2,0,2,2,1,3,2,1,3,2,1,2,3,2,2,3,0,2,3,2,2,1,2,2,2,2, -1,2,2,0,0,0,0,2,0,1,2,0,1,1,1,0,1,0,3,1,1,0,0,0,0,0,0,0,0,0,1,0, -3,3,2,3,3,2,3,2,2,2,3,2,2,3,2,2,1,2,3,2,2,3,1,3,2,2,2,3,2,2,2,3, -3,2,1,3,0,1,1,1,0,2,1,1,1,1,1,0,1,0,1,1,0,0,0,0,0,0,0,0,0,2,0,0, -1,0,0,3,0,3,3,3,3,3,0,0,3,0,2,2,3,3,3,3,3,0,0,0,1,1,3,0,0,0,0,2, -0,0,1,0,0,0,0,0,0,0,2,3,0,0,0,3,0,2,0,0,0,0,0,3,0,0,0,0,0,0,0,0, -2,0,3,3,3,3,0,0,2,3,0,0,3,0,3,3,2,3,3,3,3,3,0,0,3,3,3,0,0,0,3,3, -0,0,3,0,0,0,0,2,0,0,2,1,1,3,0,0,1,0,0,2,3,0,1,0,0,0,0,0,0,0,1,0, -3,3,3,3,2,3,3,3,3,3,3,3,1,2,1,3,3,2,2,1,2,2,2,3,1,1,2,0,2,1,2,1, -2,2,1,0,0,0,1,1,0,1,0,1,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0, -3,0,2,1,2,3,3,3,0,2,0,2,2,0,2,1,3,2,2,1,2,1,0,0,2,2,1,0,2,1,2,2, -0,1,1,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,1,3,3,1,1,3,0,2,3,1,1,3,2,1,1,2,0,2,2,3,2,1,1,1,1,1,2, -3,0,0,1,3,1,2,1,2,0,3,0,0,0,1,0,3,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0, -3,3,1,1,3,2,3,3,3,1,3,2,1,3,2,1,3,2,2,2,2,1,3,3,1,2,1,3,1,2,3,0, -2,1,1,3,2,2,2,1,2,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2, -3,3,2,3,2,3,3,2,3,2,3,2,3,3,2,1,0,3,2,2,2,1,2,2,2,1,2,2,1,2,1,1, -2,2,2,3,0,1,3,1,1,1,1,0,1,1,0,2,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,3,2,2,1,1,3,2,3,2,3,2,0,3,2,2,1,2,0,2,2,2,1,2,2,2,2,1, -3,2,1,2,2,1,0,2,0,1,0,0,1,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,1, -3,3,3,3,3,2,3,1,2,3,3,2,2,3,0,1,1,2,0,3,3,2,2,3,0,1,1,3,0,0,0,0, -3,1,0,3,3,0,2,0,2,1,0,0,3,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,3,2,3,3,0,1,3,1,1,2,1,2,1,1,3,1,1,0,2,3,1,1,1,1,1,1,1,1, -3,1,1,2,2,2,2,1,1,1,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -3,2,2,1,1,2,1,3,3,2,3,2,2,3,2,2,3,1,2,2,1,2,0,3,2,1,2,2,2,2,2,1, -3,2,1,2,2,2,1,1,1,1,0,0,1,1,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,1,3,3,0,2,1,0,3,2,0,0,3,1,0,1,1,0,1,0,0,0,0,0,1, -1,0,0,1,0,3,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,2,2,2,3,0,0,1,3,0,3,2,0,3,2,2,3,3,3,3,3,1,0,2,2,2,0,2,2,1,2, -0,2,3,0,0,0,0,1,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -3,0,2,3,1,3,3,2,3,3,0,3,3,0,3,2,2,3,2,3,3,3,0,0,2,2,3,0,1,1,1,3, -0,0,3,0,0,0,2,2,0,1,3,0,1,2,2,2,3,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1, -3,2,3,3,2,0,3,3,2,2,3,1,3,2,1,3,2,0,1,2,2,0,2,3,2,1,0,3,0,0,0,0, -3,0,0,2,3,1,3,0,0,3,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,1,3,2,2,2,1,2,0,1,3,1,1,3,1,3,0,0,2,1,1,1,1,2,1,1,1,0,2,1,0,1, -1,2,0,0,0,3,1,1,0,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,0,3,1,0,0,0,1,0, -3,3,3,3,2,2,2,2,2,1,3,1,1,1,2,0,1,1,2,1,2,1,3,2,0,0,3,1,1,1,1,1, -3,1,0,2,3,0,0,0,3,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,2,3,0,3,3,0,2,0,0,0,0,0,0,0,3,0,0,1,0,0,0,0,0,0,0,0,0,0,0, -0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,2,3,1,3,0,0,1,2,0,0,2,0,3,3,2,3,3,3,2,3,0,0,2,2,2,0,0,0,2,2, -0,0,1,0,0,0,0,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -0,0,0,3,0,2,0,0,0,0,0,0,0,0,0,0,1,2,3,1,3,3,0,0,1,0,3,0,0,0,0,0, -0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,1,2,3,1,2,3,1,0,3,0,2,2,1,0,2,1,1,2,0,1,0,0,1,1,1,1,0,1,0,0, -1,0,0,0,0,1,1,0,3,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,2,1,0,1,1,1,3,1,2,2,2,2,2,2,1,1,1,1,0,3,1,0,1,3,1,1,1,1, -1,1,0,2,0,1,3,1,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2,0,1, -3,0,2,2,1,3,3,2,3,3,0,1,1,0,2,2,1,2,1,3,3,1,0,0,3,2,0,0,0,0,2,1, -0,1,0,0,0,0,1,2,0,1,1,3,1,1,2,2,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0, -0,0,3,0,0,1,0,0,0,3,0,0,3,0,3,1,0,1,1,1,3,2,0,0,0,3,0,0,0,0,2,0, -0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0, -3,3,1,3,2,1,3,3,1,2,2,0,1,2,1,0,1,2,0,0,0,0,0,3,0,0,0,3,0,0,0,0, -3,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,1,2,0,3,3,3,2,2,0,1,1,0,1,3,0,0,0,2,2,0,0,0,0,3,1,0,1,0,0,0, -0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,2,3,1,2,0,0,2,1,0,3,1,0,1,2,0,1,1,1,1,3,0,0,3,1,1,0,2,2,1,1, -0,2,0,0,0,0,0,1,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,0,3,1,2,0,0,2,2,0,1,2,0,1,0,1,3,1,2,1,0,0,0,2,0,3,0,0,0,1,0, -0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,1,1,2,2,0,0,0,2,0,2,1,0,1,1,0,1,1,1,2,1,0,0,1,1,1,0,2,1,1,1, -0,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1, -0,0,0,2,0,1,3,1,1,1,1,0,0,0,0,3,2,0,1,0,0,0,1,2,0,0,0,1,0,0,0,0, -0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,3,3,3,3,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,0,2,3,2,2,0,0,0,1,0,0,0,0,2,3,2,1,2,2,3,0,0,0,2,3,1,0,0,0,1,1, -0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0, -3,3,2,2,0,1,0,0,0,0,2,0,2,0,1,0,0,0,1,1,0,0,0,2,1,0,1,0,1,1,0,0, -0,1,0,2,0,0,1,0,3,0,1,0,0,0,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,1,0,0,1,0,0,0,0,0,1,1,2,0,0,0,0,1,0,0,1,3,1,0,0,0,0,1,1,0,0, -0,1,0,0,0,0,3,0,0,0,0,0,0,3,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0, -3,3,1,1,1,1,2,3,0,0,2,1,1,1,1,1,0,2,1,1,0,0,0,2,1,0,1,2,1,1,0,1, -2,1,0,3,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,3,1,0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1, -0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,0,0,0,0,0,0,1,2,1,0,1,1,0,2,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,2,0,0,0,1,3,0,1,0,0,0,2,0,0,0,0,0,0,0,1,2,0,0,0,0,0, -3,3,0,0,1,1,2,0,0,1,2,1,0,1,1,1,0,1,1,0,0,2,1,1,0,1,0,0,1,1,1,0, -0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,1,0,0,0,0,1,0,0,0,0,3,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,3,0,0,1,1,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -1,1,0,1,2,0,1,2,0,0,1,1,0,2,0,1,0,0,1,0,0,0,0,1,0,0,0,2,0,0,0,0, -1,0,0,1,0,1,1,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,1,0,0,0,0,0,0,0,1,1,0,1,1,0,2,1,3,0,0,0,0,1,1,0,0,0,0,0,0,0,3, -1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,1,0,1,0,0,2,0,0,2,0,0,1,1,2,0,0,1,1,0,0,0,1,0,0,0,1,1,0,0,0, -1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0, -1,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,1,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,3,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,0,0,0, -1,0,0,0,0,0,0,0,0,1,0,0,0,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,1,1,0,0,2,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -}; - - -const SequenceModel TIS620ThaiModel = -{ - TIS620CharToOrderMap, - ThaiLangModel, - (float)0.926386, - PR_FALSE, - "TIS-620", - "Thai" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangTurkishModel.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangTurkishModel.cpp deleted file mode 100644 index 88e22c9..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/LangTurkishModel.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsSBCharSetProber.h" - -static const unsigned char turkish_ISO_8859_9CharToOrderMap[] = -{ -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, -255, 23, 37, 47, 39, 29, 52, 36, 45, 53, 60, 16, 49, 20, 46, 42, - 48, 69, 44, 35, 31, 51, 38, 62, 65, 43, 56,255,255,255,255,255, -255, 1, 21, 28, 12, 2, 18, 27, 25, 3, 24, 10, 5, 13, 4, 15, - 26, 64, 7, 8, 9, 14, 32, 57, 58, 11, 22,255,255,255,255,255, -180,179,178,177,176,175,174,173,172,171,170,169,168,167,166,165, -164,163,162,161,160,159,101,158,157,156,155,154,153,152,151,106, -150,149,148,147,146,145,144,100,143,142,141,140,139,138,137,136, - 94, 80, 93,135,105,134,133, 63,132,131,130,129,128,127,126,125, -124,104, 73, 99, 79, 85,123, 54,122, 98, 92,121,120, 91,103,119, - 68,118,117, 97,116,115, 50, 90,114,113,112,111, 55, 41, 40, 86, - 89, 70, 59, 78, 71, 82, 88, 33, 77, 66, 84, 83,110, 75, 61, 96, - 30, 67,109, 74, 87,102, 34, 95, 81,108, 76, 72, 17, 6, 19,107, -}; - -static const PRUint8 turkishLangModel[] = -{ -3,2,3,3,3,1,3,3,3,3,3,3,3,3,2,1,1,3,3,1,3,3,0,3,3,3,3,3,0,3,1,3, -3,2,1,0,0,1,1,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0,2,2,0,0,1,0,0,1, -3,2,2,3,3,0,3,3,3,3,3,3,3,2,3,1,0,3,3,1,3,3,0,3,3,3,3,3,0,3,0,3, -3,1,1,0,1,0,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,2,2,0,0,0,1,0,1, -3,3,2,3,3,0,3,3,3,3,3,3,3,2,3,1,1,3,3,0,3,3,1,2,3,3,3,3,0,3,0,3, -3,1,1,0,0,0,1,0,0,0,0,1,1,0,1,2,1,0,0,0,1,0,0,0,0,2,0,0,0,0,0,1, -3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,1,3,3,2,0,3,2,1,2,2,1,3,3,0,0,0,2, -2,2,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,1,0,0,1, -3,3,3,2,3,3,1,2,3,3,3,3,3,3,3,1,3,2,1,0,3,2,0,1,2,3,3,2,1,0,0,2, -2,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2,0,2,0,0,0, -1,0,1,3,3,1,3,3,3,3,3,3,3,1,2,0,0,2,3,0,2,3,0,0,2,2,2,3,0,3,0,1, -2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,0,3,2,0,2,3,2,3,3,1,0,0,2, -3,2,0,0,1,0,0,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,1,1,1,0,2,0,0,1, -3,3,3,2,3,3,2,3,3,3,3,2,3,3,3,0,3,3,0,0,2,1,0,0,2,3,2,2,0,0,0,2, -2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,1,0,2,0,0,1, -3,3,3,2,3,3,3,3,3,3,3,2,3,3,3,0,3,2,0,1,3,2,1,1,3,2,3,2,1,0,0,2, -2,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0, -3,3,3,2,3,3,3,3,3,3,3,2,3,3,3,0,3,2,2,0,2,3,0,0,2,2,2,2,0,0,0,2, -3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,1,0,0,0, -3,3,3,3,3,3,3,2,2,2,2,3,2,3,3,0,3,3,1,1,2,2,0,0,2,2,3,2,0,0,1,3, -0,3,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1, -3,3,3,2,3,3,3,2,1,2,2,3,2,3,3,0,3,2,0,0,1,1,0,1,1,2,1,2,0,0,0,1, -0,3,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0, -3,3,3,2,3,3,2,3,2,2,2,3,3,3,3,1,3,1,1,0,3,2,1,1,3,3,2,3,1,0,0,1, -1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,1, -3,2,2,3,3,0,3,3,3,3,3,3,3,2,2,1,0,3,3,1,3,3,0,1,3,3,2,3,0,3,0,3, -2,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0, -2,2,2,3,3,0,3,3,3,3,3,3,3,3,3,0,0,3,2,0,3,3,0,3,2,3,3,3,0,3,1,3, -2,0,0,0,0,0,0,0,0,0,0,1,0,1,2,0,1,0,0,0,0,0,0,0,2,2,0,0,1,0,0,1, -3,3,3,1,2,3,3,1,0,0,1,0,0,3,3,2,3,0,0,2,0,0,2,0,2,0,0,0,2,0,2,0, -0,3,1,0,1,0,0,0,2,2,1,0,1,1,2,1,2,2,2,0,2,1,1,0,0,0,2,0,0,0,0,0, -1,2,1,3,3,0,3,3,3,3,3,2,3,0,0,0,0,2,3,0,2,3,1,0,2,3,1,3,0,3,0,2, -3,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,1,3,3,2,2,3,2,2,0,1,2,3,0,1,2,1,0,1,0,0,0,1,0,2,2,0,0,0,1, -1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0, -3,3,3,1,3,3,1,1,3,3,1,1,3,3,1,0,2,1,2,0,2,1,0,0,1,1,2,1,0,0,0,2, -2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,1,0,2,1,3,0,0,2,0,0,3,3,0,3,0,0,1,0,1,2,0,0,1,1,2,2,0,1,0, -0,1,2,1,1,0,1,0,1,1,1,1,1,0,1,1,1,2,2,1,2,0,1,0,0,0,0,0,0,1,0,0, -3,3,3,2,3,2,3,3,0,2,2,2,3,3,3,0,3,0,0,0,2,2,0,1,2,1,1,1,0,0,0,1, -0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0, -3,3,3,3,3,3,2,1,2,2,3,3,3,3,2,0,2,0,0,0,2,2,0,0,2,1,3,3,0,0,1,1, -1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0, -1,1,2,3,3,0,3,3,3,3,3,3,2,2,0,2,0,2,3,2,3,2,2,2,2,2,2,2,1,3,2,3, -2,0,2,1,2,2,2,2,1,1,2,2,1,2,2,1,2,0,0,2,1,1,0,2,1,0,0,1,0,0,0,1, -2,3,3,1,1,1,0,1,1,1,2,3,2,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0, -0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,2,2,3,2,3,2,2,1,3,3,3,0,2,1,2,0,2,1,0,0,1,1,1,1,1,0,0,1, -2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,2,0,1,0,0,0, -3,3,3,2,3,3,3,3,3,2,3,1,2,3,3,1,2,0,0,0,0,0,0,0,3,2,1,1,0,0,0,0, -2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0, -3,3,3,2,2,3,3,2,1,1,1,1,1,3,3,0,3,1,0,0,1,1,0,0,3,1,2,1,0,0,0,0, -0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0, -3,3,3,2,2,3,2,2,2,3,2,1,1,3,3,0,3,0,0,0,0,1,0,0,3,1,1,2,0,0,0,1, -1,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, -1,1,1,3,3,0,3,3,3,3,3,2,2,2,1,2,0,2,1,2,2,1,1,0,1,2,2,2,2,2,2,2, -0,0,2,1,2,1,2,1,0,1,1,3,1,2,1,1,2,0,0,2,0,1,0,1,0,1,0,0,0,1,0,1, -3,3,3,1,3,3,3,0,1,1,0,2,2,3,1,0,3,0,0,0,1,0,0,0,1,0,0,1,0,1,0,0, -1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,2,0,0,2,2,1,0,0,1,0,0,3,3,1,3,0,0,1,1,0,2,0,3,0,0,0,2,0,1,1, -0,1,2,0,1,2,2,0,2,2,2,2,1,0,2,1,1,0,2,0,2,1,2,0,0,0,0,0,0,0,0,0, -3,3,3,1,3,2,3,2,0,2,2,2,1,3,2,0,2,1,2,0,1,2,0,0,1,0,2,2,0,0,0,2, -1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0, -3,3,3,0,3,3,1,1,2,3,1,0,3,2,3,0,3,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0, -1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,3,3,0,3,3,2,3,3,2,2,0,0,0,0,1,2,0,1,3,0,0,0,3,1,1,0,3,0,2, -2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,1,2,2,1,0,3,1,1,1,1,3,3,2,3,0,0,1,0,1,2,0,2,2,0,2,2,0,2,1, -0,2,2,1,1,1,1,0,2,1,1,0,1,1,1,1,2,1,2,1,2,0,1,0,1,0,0,0,0,0,0,0, -3,3,3,0,1,1,3,0,0,1,1,0,0,2,2,0,3,0,0,1,1,0,1,0,0,0,0,0,2,0,0,0, -0,3,1,0,1,0,1,0,2,0,0,1,0,1,0,1,1,1,2,1,1,0,2,0,0,0,0,0,0,0,0,0, -3,3,3,0,2,0,2,0,1,1,1,0,0,3,3,0,2,0,0,1,0,0,2,1,1,0,1,0,1,0,1,0, -0,2,0,1,2,0,2,0,2,1,1,0,1,0,2,1,1,0,2,1,1,0,1,0,0,0,1,1,0,0,0,0, -3,2,3,0,1,0,0,0,0,0,0,0,0,1,2,0,1,0,0,1,0,0,1,0,0,0,0,0,2,0,0,0, -0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,2,1,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,0,2,3,0,0,1,0,1,0,2,3,2,3,0,0,1,3,0,2,1,0,0,0,0,2,0,1,0, -0,2,1,0,0,1,1,0,2,1,0,0,1,0,0,1,1,0,1,1,2,0,1,0,0,0,0,1,0,0,0,0, -3,2,2,0,0,1,1,0,0,0,0,0,0,3,1,1,1,0,0,0,0,0,1,0,0,0,0,0,2,0,1,0, -0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0, -0,0,0,3,3,0,2,3,2,2,1,2,2,1,1,2,0,1,3,2,2,2,0,0,2,2,0,0,0,1,2,1, -3,0,2,1,1,0,1,1,1,0,1,2,2,2,1,1,2,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0, -0,1,1,2,3,0,3,3,3,2,2,2,2,1,0,1,0,1,0,1,2,2,0,0,2,2,1,3,1,1,2,1, -0,0,1,1,2,0,1,1,0,0,1,2,0,2,1,1,2,0,0,1,0,0,0,1,0,1,0,1,0,0,0,0, -3,3,2,0,0,3,1,0,0,0,0,0,0,3,2,1,2,0,0,1,0,0,2,0,0,0,0,0,2,0,1,0, -0,2,1,1,0,0,1,0,1,2,0,0,1,1,0,0,2,1,1,1,1,0,2,0,0,0,0,0,0,0,0,0, -3,3,2,0,0,1,0,0,0,0,1,0,0,3,3,2,2,0,0,1,0,0,2,0,1,0,0,0,2,0,1,0, -0,0,1,1,0,0,2,0,2,1,0,0,1,1,2,1,2,0,2,1,2,1,1,1,0,0,1,1,0,0,0,0, -3,3,2,0,0,2,2,0,0,0,1,1,0,2,2,1,3,1,0,1,0,1,2,0,0,0,0,0,1,0,1,0, -0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,2,0,0,0,1,0,0,1,0,0,2,3,1,2,0,0,1,0,0,2,0,0,0,1,0,2,0,2,0, -0,1,1,2,2,1,2,0,2,1,1,0,0,1,1,0,1,1,1,1,2,1,1,0,0,0,0,0,0,0,0,0, -3,3,3,0,2,1,2,1,0,0,1,1,0,3,3,1,2,0,0,1,0,0,2,0,2,0,1,1,2,0,0,0, -0,0,1,1,1,1,2,0,1,1,0,1,1,1,1,0,0,0,1,1,1,0,1,0,0,0,1,0,0,0,0,0, -3,3,3,0,2,2,3,2,0,0,1,0,0,2,3,1,0,0,0,0,0,0,2,0,2,0,0,0,2,0,0,0, -0,1,1,0,0,0,1,0,0,1,0,1,1,0,1,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0, -3,2,3,0,0,0,0,0,0,0,1,0,0,2,2,2,2,0,0,1,0,0,2,0,0,0,0,0,2,0,1,0, -0,0,2,1,1,0,1,0,2,1,1,0,0,1,1,2,1,0,2,0,2,0,1,0,0,0,2,0,0,0,0,0, -0,0,0,2,2,0,2,1,1,1,1,2,2,0,0,1,0,1,0,0,1,3,0,0,0,0,1,0,0,2,1,0, -0,0,1,0,1,0,0,0,0,0,2,1,0,1,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0, -2,0,0,2,3,0,2,3,1,2,2,0,2,0,0,2,0,2,1,1,1,2,1,0,0,1,2,1,1,2,1,0, -1,0,2,0,1,0,1,1,0,0,2,2,1,2,1,1,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0, -3,3,3,0,2,1,2,0,0,0,1,0,0,3,2,0,1,0,0,1,0,0,2,0,0,0,1,2,1,0,1,0, -0,0,0,0,1,0,1,0,0,1,0,0,0,0,1,0,1,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0, -0,0,0,2,2,0,2,2,1,1,0,1,1,1,1,1,0,0,1,2,1,1,1,0,1,0,0,0,1,1,1,1, -0,0,2,1,0,1,1,1,0,1,1,2,1,2,1,1,2,0,1,1,2,1,0,2,0,0,0,0,0,0,0,0, -3,2,2,0,0,2,0,0,0,0,0,0,0,2,2,0,2,0,0,1,0,0,2,0,0,0,0,0,2,0,0,0, -0,2,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0, -0,0,0,3,2,0,2,2,0,1,1,0,1,0,0,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0, -2,0,1,0,1,0,1,1,0,0,1,2,0,1,0,1,1,0,0,1,0,1,0,2,0,0,0,0,0,0,0,0, -2,2,2,0,1,1,0,0,0,1,0,0,0,1,2,0,1,0,0,1,0,0,1,0,0,0,0,1,2,0,1,0, -0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,2,1,0,1,1,1,0,0,0,0,1,2,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0, -1,1,2,0,1,0,0,0,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,0,1, -0,0,1,2,2,0,2,1,2,1,1,2,2,0,0,0,0,1,0,0,1,1,0,0,2,0,0,0,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0, -2,2,2,0,0,0,1,0,0,0,0,0,0,2,2,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0, -0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -2,2,2,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,1,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, -}; - -const SequenceModel ISO_8859_9turkishModel = -{ - turkish_ISO_8859_9CharToOrderMap, - turkishLangModel, - (float)0.970290, - PR_TRUE, - "ISO-8859-9", - "turkish" -}; diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/Makefile.in b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/Makefile.in deleted file mode 100644 index 8c7f9a1..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/Makefile.in +++ /dev/null @@ -1,84 +0,0 @@ -# -# ***** BEGIN LICENSE BLOCK ***** -# Version: MPL 1.1/GPL 2.0/LGPL 2.1 -# -# The contents of this file are subject to the Mozilla Public License Version -# 1.1 (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.mozilla.org/MPL/ -# -# Software distributed under the License is distributed on an "AS IS" basis, -# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -# for the specific language governing rights and limitations under the -# License. -# -# The Original Code is Mozilla Universal charset detector code. -# -# The Initial Developer of the Original Code is -# Netscape Communications Corporation. -# Portions created by the Initial Developer are Copyright (C) 1998 -# the Initial Developer. All Rights Reserved. -# -# Contributor(s): -# Simon Montagu -# Shy Shalom -# Proofpoint, Inc. -# -# Alternatively, the contents of this file may be used under the terms of -# either the GNU General Public License Version 2 or later (the "GPL"), or -# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), -# in which case the provisions of the GPL or the LGPL are applicable instead -# of those above. If you wish to allow use of your version of this file only -# under the terms of either the GPL or the LGPL, and not to allow others to -# use your version of this file under the terms of the MPL, indicate your -# decision by deleting the provisions above and replace them with the notice -# and other provisions required by the GPL or the LGPL. If you do not delete -# the provisions above, a recipient may use your version of this file under -# the terms of any one of the MPL, the GPL or the LGPL. -# -# ***** END LICENSE BLOCK ***** - -DEPTH = ../../../.. -topsrcdir = @top_srcdir@ -srcdir = @srcdir@ -VPATH = @srcdir@ - -include $(DEPTH)/config/autoconf.mk - -MODULE = universalchardet_s -LIBRARY_NAME = universalchardet_s - -CPPSRCS = \ - CharDistribution.cpp \ - JpCntx.cpp \ - LangBulgarianModel.cpp \ - LangCyrillicModel.cpp \ - LangGreekModel.cpp \ - LangHungarianModel.cpp \ - LangHebrewModel.cpp \ - LangThaiModel.cpp \ - nsHebrewProber.cpp \ - nsCharSetProber.cpp \ - nsBig5Prober.cpp \ - nsEUCJPProber.cpp \ - nsEUCKRProber.cpp \ - nsEUCTWProber.cpp \ - nsEscCharsetProber.cpp \ - nsEscSM.cpp \ - nsGB2312Prober.cpp \ - nsMBCSGroupProber.cpp \ - nsMBCSSM.cpp \ - nsSBCSGroupProber.cpp \ - nsSBCharSetProber.cpp \ - nsSJISProber.cpp \ - nsUTF8Prober.cpp \ - nsLatin1Prober.cpp \ - nsUniversalDetector.cpp \ - $(NULL) - -FORCE_STATIC_LIB = 1 -LIBXUL_LIBRARY = 1 -# This library is used by other shared libs in a static build -FORCE_USE_PIC = 1 - -include $(topsrcdir)/config/rules.mk diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsBig5Prober.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsBig5Prober.cpp deleted file mode 100644 index a5bf090..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsBig5Prober.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsBig5Prober.h" -#include "nsDebug.h" - -void nsBig5Prober::Reset(void) -{ - mCodingSM->Reset(); - mState = eDetecting; - mDistributionAnalyser.Reset(mIsPreferredLanguage); -} - -nsProbingState nsBig5Prober::HandleData(const char* aBuf, PRUint32 aLen) -{ - NS_ASSERTION(aLen, "HandleData called with empty buffer"); - nsSMState codingState; - - for (PRUint32 i = 0; i < aLen; i++) - { - codingState = mCodingSM->NextState(aBuf[i]); - if (codingState == eItsMe) - { - mState = eFoundIt; - break; - } - if (codingState == eStart) - { - PRUint32 charLen = mCodingSM->GetCurrentCharLen(); - - if (i == 0) - { - mLastChar[1] = aBuf[0]; - mDistributionAnalyser.HandleOneChar(mLastChar, charLen); - } - else - mDistributionAnalyser.HandleOneChar(aBuf+i-1, charLen); - } - } - - mLastChar[0] = aBuf[aLen-1]; - - if (mState == eDetecting) - if (mDistributionAnalyser.GotEnoughData() && GetConfidence() > SHORTCUT_THRESHOLD) - mState = eFoundIt; - - return mState; -} - -float nsBig5Prober::GetConfidence(void) -{ - float distribCf = mDistributionAnalyser.GetConfidence(); - - return (float)distribCf; -} - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsBig5Prober.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsBig5Prober.h deleted file mode 100644 index 5ae3576..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsBig5Prober.h +++ /dev/null @@ -1,74 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsBig5Prober_h__ -#define nsBig5Prober_h__ - -#include "nsCharSetProber.h" -#include "nsCodingStateMachine.h" -#include "CharDistribution.h" - -class nsBig5Prober: public nsCharSetProber { -public: - nsBig5Prober(PRBool aIsPreferredLanguage) - :mIsPreferredLanguage(aIsPreferredLanguage) - {mCodingSM = new nsCodingStateMachine(&Big5SMModel); - Reset();} - virtual ~nsBig5Prober(void){delete mCodingSM;} - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName() {return "Big5";} - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void); - void SetOpion() {} - -protected: - void GetDistribution(PRUint32 aCharLen, const char* aStr); - - nsCodingStateMachine* mCodingSM; - nsProbingState mState; - - //Big5ContextAnalysis mContextAnalyser; - Big5DistributionAnalysis mDistributionAnalyser; - char mLastChar[2]; - PRBool mIsPreferredLanguage; - -}; - - -#endif /* nsBig5Prober_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsCharSetProber.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsCharSetProber.cpp deleted file mode 100644 index 0429dd1..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsCharSetProber.cpp +++ /dev/null @@ -1,125 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 2001 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shy Shalom - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsCharSetProber.h" -#include "prmem.h" - -//This filter applies to all scripts which do not use English characters -PRBool nsCharSetProber::FilterWithoutEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen) -{ - char *newptr; - char *prevPtr, *curPtr; - - PRBool meetMSB = PR_FALSE; - newptr = *newBuf = (char*)PR_Malloc(aLen); - if (!newptr) - return PR_FALSE; - - for (curPtr = prevPtr = (char*)aBuf; curPtr < aBuf+aLen; curPtr++) - { - if (*curPtr & 0x80) - { - meetMSB = PR_TRUE; - } - else if (*curPtr < 'A' || (*curPtr > 'Z' && *curPtr < 'a') || *curPtr > 'z') - { - //current char is a symbol, most likely a punctuation. we treat it as segment delimiter - if (meetMSB && curPtr > prevPtr) - //this segment contains more than single symbol, and it has upper ASCII, we need to keep it - { - while (prevPtr < curPtr) *newptr++ = *prevPtr++; - prevPtr++; - *newptr++ = ' '; - meetMSB = PR_FALSE; - } - else //ignore current segment. (either because it is just a symbol or just an English word) - prevPtr = curPtr+1; - } - } - if (meetMSB && curPtr > prevPtr) - while (prevPtr < curPtr) *newptr++ = *prevPtr++; - - newLen = newptr - *newBuf; - - return PR_TRUE; -} - -//This filter applies to all scripts which contain both English characters and upper ASCII characters. -PRBool nsCharSetProber::FilterWithEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen) -{ - //do filtering to reduce load to probers - char *newptr; - char *prevPtr, *curPtr; - PRBool isInTag = PR_FALSE; - - newptr = *newBuf = (char*)PR_Malloc(aLen); - if (!newptr) - return PR_FALSE; - - for (curPtr = prevPtr = (char*)aBuf; curPtr < aBuf+aLen; curPtr++) - { - if (*curPtr == '>') - isInTag = PR_FALSE; - else if (*curPtr == '<') - isInTag = PR_TRUE; - - if (!(*curPtr & 0x80) && - (*curPtr < 'A' || (*curPtr > 'Z' && *curPtr < 'a') || *curPtr > 'z') ) - { - if (curPtr > prevPtr && !isInTag) // Current segment contains more than just a symbol - // and it is not inside a tag, keep it. - { - while (prevPtr < curPtr) *newptr++ = *prevPtr++; - prevPtr++; - *newptr++ = ' '; - } - else - prevPtr = curPtr+1; - } - } - - // If the current segment contains more than just a symbol - // and it is not inside a tag then keep it. - if (!isInTag) - while (prevPtr < curPtr) - *newptr++ = *prevPtr++; - - newLen = newptr - *newBuf; - - return PR_TRUE; -} diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsCharSetProber.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsCharSetProber.h deleted file mode 100644 index 436928a..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsCharSetProber.h +++ /dev/null @@ -1,78 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 2001 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shy Shalom - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -#ifndef nsCharSetProber_h__ -#define nsCharSetProber_h__ - -#include "nscore.h" - -//#define DEBUG_chardet // Uncomment this for debug dump. - -typedef enum { - eDetecting = 0, //We are still detecting, no sure answer yet, but caller can ask for confidence. - eFoundIt = 1, //That's a positive answer - eNotMe = 2 //Negative answer -} nsProbingState; - -#define SHORTCUT_THRESHOLD (float)0.95 - -class nsCharSetProber { -public: - virtual ~nsCharSetProber() {} - virtual const char* GetCharSetName() = 0; - virtual const char* GetLangName() {return "";}; - virtual PRBool KeepEnglishLetters() {return PR_TRUE;} - virtual nsProbingState HandleData(const char* aBuf, PRUint32 aLen) = 0; - virtual nsProbingState GetState(void) = 0; - virtual void Reset(void) = 0; - virtual float GetConfidence(void) = 0; - virtual void SetOpion() = 0; - -#ifdef DEBUG_chardet - virtual void DumpStatus() {}; -#endif - - // Helper functions used in the Latin1 and Group probers. - // both functions Allocate a new buffer for newBuf. This buffer should be - // freed by the caller using PR_FREEIF. - // Both functions return PR_FALSE in case of memory allocation failure. - static PRBool FilterWithoutEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen); - static PRBool FilterWithEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen); - -}; - -#endif /* nsCharSetProber_h__ */ diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsCodingStateMachine.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsCodingStateMachine.h deleted file mode 100644 index 819f9ab..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsCodingStateMachine.h +++ /dev/null @@ -1,104 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -#ifndef nsCodingStateMachine_h__ -#define nsCodingStateMachine_h__ - -#include "nsPkgInt.h" - -typedef enum { - eStart = 0, - eError = 1, - eItsMe = 2 -} nsSMState; - -#define GETCLASS(c) GETFROMPCK(((unsigned char)(c)), mModel->classTable) - -//state machine model -typedef struct -{ - nsPkgInt classTable; - PRUint32 classFactor; - nsPkgInt stateTable; - const PRUint32* charLenTable; - const char* name; -} SMModel; - -class nsCodingStateMachine { -public: - nsCodingStateMachine(const SMModel* sm) : mModel(sm) { mCurrentState = eStart; } - nsSMState NextState(char c){ - //for each byte we get its class , if it is first byte, we also get byte length - PRUint32 byteCls = GETCLASS(c); - if (mCurrentState == eStart) - { - mCurrentBytePos = 0; - mCurrentCharLen = mModel->charLenTable[byteCls]; - } - //from byte's class and stateTable, we get its next state - mCurrentState=(nsSMState)GETFROMPCK(mCurrentState*(mModel->classFactor)+byteCls, - mModel->stateTable); - mCurrentBytePos++; - return mCurrentState; - } - PRUint32 GetCurrentCharLen(void) {return mCurrentCharLen;} - void Reset(void) {mCurrentState = eStart;} - const char * GetCodingStateMachine() {return mModel->name;} - -protected: - nsSMState mCurrentState; - PRUint32 mCurrentCharLen; - PRUint32 mCurrentBytePos; - - const SMModel *mModel; -}; - -extern const SMModel UTF8SMModel; -extern const SMModel Big5SMModel; -extern const SMModel EUCJPSMModel; -extern const SMModel EUCKRSMModel; -extern const SMModel EUCTWSMModel; -extern const SMModel GB18030SMModel; -extern const SMModel SJISSMModel; - - -extern const SMModel HZSMModel; -extern const SMModel ISO2022CNSMModel; -extern const SMModel ISO2022JPSMModel; -extern const SMModel ISO2022KRSMModel; - -#endif /* nsCodingStateMachine_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCJPProber.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCJPProber.cpp deleted file mode 100644 index a89ee22..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCJPProber.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -// for japanese encoding, obeserve characteristic: -// 1, kana character (or hankaku?) often have hight frequency of appereance -// 2, kana character often exist in group -// 3, certain combination of kana is never used in japanese language - -#include "nsEUCJPProber.h" -#include "nsDebug.h" - -void nsEUCJPProber::Reset(void) -{ - mCodingSM->Reset(); - mState = eDetecting; - mContextAnalyser.Reset(mIsPreferredLanguage); - mDistributionAnalyser.Reset(mIsPreferredLanguage); -} - -nsProbingState nsEUCJPProber::HandleData(const char* aBuf, PRUint32 aLen) -{ - NS_ASSERTION(aLen, "HandleData called with empty buffer"); - nsSMState codingState; - - for (PRUint32 i = 0; i < aLen; i++) - { - codingState = mCodingSM->NextState(aBuf[i]); - if (codingState == eItsMe) - { - mState = eFoundIt; - break; - } - if (codingState == eStart) - { - PRUint32 charLen = mCodingSM->GetCurrentCharLen(); - - if (i == 0) - { - mLastChar[1] = aBuf[0]; - mContextAnalyser.HandleOneChar(mLastChar, charLen); - mDistributionAnalyser.HandleOneChar(mLastChar, charLen); - } - else - { - mContextAnalyser.HandleOneChar(aBuf+i-1, charLen); - mDistributionAnalyser.HandleOneChar(aBuf+i-1, charLen); - } - } - } - - mLastChar[0] = aBuf[aLen-1]; - - if (mState == eDetecting) - if (mContextAnalyser.GotEnoughData() && GetConfidence() > SHORTCUT_THRESHOLD) - mState = eFoundIt; - - return mState; -} - -float nsEUCJPProber::GetConfidence(void) -{ - float contxtCf = mContextAnalyser.GetConfidence(); - float distribCf = mDistributionAnalyser.GetConfidence(); - - return (contxtCf > distribCf ? contxtCf : distribCf); -} - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCJPProber.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCJPProber.h deleted file mode 100644 index a7a2f51..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCJPProber.h +++ /dev/null @@ -1,78 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -// for S-JIS encoding, obeserve characteristic: -// 1, kana character (or hankaku?) often have hight frequency of appereance -// 2, kana character often exist in group -// 3, certain combination of kana is never used in japanese language - -#ifndef nsEUCJPProber_h__ -#define nsEUCJPProber_h__ - -#include "nsCharSetProber.h" -#include "nsCodingStateMachine.h" -#include "JpCntx.h" -#include "CharDistribution.h" - -class nsEUCJPProber: public nsCharSetProber { -public: - nsEUCJPProber(PRBool aIsPreferredLanguage) - :mIsPreferredLanguage(aIsPreferredLanguage) - {mCodingSM = new nsCodingStateMachine(&EUCJPSMModel); - Reset();} - virtual ~nsEUCJPProber(void){delete mCodingSM;} - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName() {return "EUC-JP";} - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void); - void SetOpion() {} - -protected: - nsCodingStateMachine* mCodingSM; - nsProbingState mState; - - EUCJPContextAnalysis mContextAnalyser; - EUCJPDistributionAnalysis mDistributionAnalyser; - - char mLastChar[2]; - PRBool mIsPreferredLanguage; -}; - - -#endif /* nsEUCJPProber_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCKRProber.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCKRProber.cpp deleted file mode 100644 index 76e3217..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCKRProber.cpp +++ /dev/null @@ -1,93 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsEUCKRProber.h" -#include "nsDebug.h" - -void nsEUCKRProber::Reset(void) -{ - mCodingSM->Reset(); - mState = eDetecting; - mDistributionAnalyser.Reset(mIsPreferredLanguage); - //mContextAnalyser.Reset(); -} - -nsProbingState nsEUCKRProber::HandleData(const char* aBuf, PRUint32 aLen) -{ - NS_ASSERTION(aLen, "HandleData called with empty buffer"); - nsSMState codingState; - - for (PRUint32 i = 0; i < aLen; i++) - { - codingState = mCodingSM->NextState(aBuf[i]); - if (codingState == eItsMe) - { - mState = eFoundIt; - break; - } - if (codingState == eStart) - { - PRUint32 charLen = mCodingSM->GetCurrentCharLen(); - - if (i == 0) - { - mLastChar[1] = aBuf[0]; - mDistributionAnalyser.HandleOneChar(mLastChar, charLen); - } - else - mDistributionAnalyser.HandleOneChar(aBuf+i-1, charLen); - } - } - - mLastChar[0] = aBuf[aLen-1]; - - if (mState == eDetecting) - if (mDistributionAnalyser.GotEnoughData() && GetConfidence() > SHORTCUT_THRESHOLD) - mState = eFoundIt; -// else -// mDistributionAnalyser.HandleData(aBuf, aLen); - - return mState; -} - -float nsEUCKRProber::GetConfidence(void) -{ - float distribCf = mDistributionAnalyser.GetConfidence(); - - return (float)distribCf; -} - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCKRProber.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCKRProber.h deleted file mode 100644 index 8e09984..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCKRProber.h +++ /dev/null @@ -1,75 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsEUCKRProber_h__ -#define nsEUCKRProber_h__ - -#include "nsCharSetProber.h" -#include "nsCodingStateMachine.h" -#include "CharDistribution.h" - -class nsEUCKRProber: public nsCharSetProber { -public: - nsEUCKRProber(PRBool aIsPreferredLanguage) - :mIsPreferredLanguage(aIsPreferredLanguage) - {mCodingSM = new nsCodingStateMachine(&EUCKRSMModel); - Reset(); - } - virtual ~nsEUCKRProber(void){delete mCodingSM;} - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName() {return "EUC-KR";} - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void); - void SetOpion() {} - -protected: - void GetDistribution(PRUint32 aCharLen, const char* aStr); - - nsCodingStateMachine* mCodingSM; - nsProbingState mState; - - //EUCKRContextAnalysis mContextAnalyser; - EUCKRDistributionAnalysis mDistributionAnalyser; - char mLastChar[2]; - PRBool mIsPreferredLanguage; - -}; - - -#endif /* nsEUCKRProber_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCTWProber.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCTWProber.cpp deleted file mode 100644 index 73d552a..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCTWProber.cpp +++ /dev/null @@ -1,93 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsEUCTWProber.h" -#include "nsDebug.h" - -void nsEUCTWProber::Reset(void) -{ - mCodingSM->Reset(); - mState = eDetecting; - mDistributionAnalyser.Reset(mIsPreferredLanguage); - //mContextAnalyser.Reset(); -} - -nsProbingState nsEUCTWProber::HandleData(const char* aBuf, PRUint32 aLen) -{ - NS_ASSERTION(aLen, "HandleData called with empty buffer"); - nsSMState codingState; - - for (PRUint32 i = 0; i < aLen; i++) - { - codingState = mCodingSM->NextState(aBuf[i]); - if (codingState == eItsMe) - { - mState = eFoundIt; - break; - } - if (codingState == eStart) - { - PRUint32 charLen = mCodingSM->GetCurrentCharLen(); - - if (i == 0) - { - mLastChar[1] = aBuf[0]; - mDistributionAnalyser.HandleOneChar(mLastChar, charLen); - } - else - mDistributionAnalyser.HandleOneChar(aBuf+i-1, charLen); - } - } - - mLastChar[0] = aBuf[aLen-1]; - - if (mState == eDetecting) - if (mDistributionAnalyser.GotEnoughData() && GetConfidence() > SHORTCUT_THRESHOLD) - mState = eFoundIt; -// else -// mDistributionAnalyser.HandleData(aBuf, aLen); - - return mState; -} - -float nsEUCTWProber::GetConfidence(void) -{ - float distribCf = mDistributionAnalyser.GetConfidence(); - - return (float)distribCf; -} - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCTWProber.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCTWProber.h deleted file mode 100644 index d7acd40..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEUCTWProber.h +++ /dev/null @@ -1,75 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsEUCTWProber_h__ -#define nsEUCTWProber_h__ - -#include "nsCharSetProber.h" -#include "nsCodingStateMachine.h" -#include "CharDistribution.h" - -class nsEUCTWProber: public nsCharSetProber { -public: - nsEUCTWProber(PRBool aIsPreferredLanguage) - :mIsPreferredLanguage(aIsPreferredLanguage) - {mCodingSM = new nsCodingStateMachine(&EUCTWSMModel); - Reset();} - virtual ~nsEUCTWProber(void){delete mCodingSM;} - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName() {return "EUC-TW";} - //const char* GetCharSetName() {return "x-euc-tw";} - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void); - void SetOpion() {} - -protected: - void GetDistribution(PRUint32 aCharLen, const char* aStr); - - nsCodingStateMachine* mCodingSM; - nsProbingState mState; - - //EUCTWContextAnalysis mContextAnalyser; - EUCTWDistributionAnalysis mDistributionAnalyser; - char mLastChar[2]; - PRBool mIsPreferredLanguage; - -}; - - -#endif /* nsEUCTWProber_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEscCharsetProber.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEscCharsetProber.cpp deleted file mode 100644 index 464c753..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEscCharsetProber.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - - -#include "nsEscCharsetProber.h" -#include "nsUniversalDetector.h" - -nsEscCharSetProber::nsEscCharSetProber(PRUint32 aLanguageFilter) -{ - for (PRUint32 i = 0; i < NUM_OF_ESC_CHARSETS; i++) - mCodingSM[i] = nsnull; - if (aLanguageFilter & NS_FILTER_CHINESE_SIMPLIFIED) - { - mCodingSM[0] = new nsCodingStateMachine(&HZSMModel); - mCodingSM[1] = new nsCodingStateMachine(&ISO2022CNSMModel); - } - if (aLanguageFilter & NS_FILTER_JAPANESE) - mCodingSM[2] = new nsCodingStateMachine(&ISO2022JPSMModel); - if (aLanguageFilter & NS_FILTER_KOREAN) - mCodingSM[3] = new nsCodingStateMachine(&ISO2022KRSMModel); - mActiveSM = NUM_OF_ESC_CHARSETS; - mState = eDetecting; - mDetectedCharset = nsnull; -} - -nsEscCharSetProber::~nsEscCharSetProber(void) -{ - for (PRUint32 i = 0; i < NUM_OF_ESC_CHARSETS; i++) - delete mCodingSM[i]; -} - -void nsEscCharSetProber::Reset(void) -{ - mState = eDetecting; - for (PRUint32 i = 0; i < NUM_OF_ESC_CHARSETS; i++) - if (mCodingSM[i]) - mCodingSM[i]->Reset(); - mActiveSM = NUM_OF_ESC_CHARSETS; - mDetectedCharset = nsnull; -} - -nsProbingState nsEscCharSetProber::HandleData(const char* aBuf, PRUint32 aLen) -{ - nsSMState codingState; - PRInt32 j; - PRUint32 i; - - for ( i = 0; i < aLen && mState == eDetecting; i++) - { - for (j = mActiveSM-1; j>= 0; j--) - { - if (mCodingSM[j]) - { - codingState = mCodingSM[j]->NextState(aBuf[i]); - if (codingState == eItsMe) - { - mState = eFoundIt; - mDetectedCharset = mCodingSM[j]->GetCodingStateMachine(); - return mState; - } - } - } - } - - return mState; -} - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEscCharsetProber.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEscCharsetProber.h deleted file mode 100644 index 4b648e0..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEscCharsetProber.h +++ /dev/null @@ -1,67 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsEscCharSetProber_h__ -#define nsEscCharSetProber_h__ - -#include "nsCharSetProber.h" -#include "nsCodingStateMachine.h" - -#define NUM_OF_ESC_CHARSETS 4 - -class nsEscCharSetProber: public nsCharSetProber { -public: - nsEscCharSetProber(PRUint32 aLanguageFilter); - virtual ~nsEscCharSetProber(void); - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName() {return mDetectedCharset;} - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void){return (float)0.99;} - void SetOpion() {} - -protected: - void GetDistribution(PRUint32 aCharLen, const char* aStr); - - nsCodingStateMachine* mCodingSM[NUM_OF_ESC_CHARSETS] ; - PRUint32 mActiveSM; - nsProbingState mState; - const char * mDetectedCharset; -}; - -#endif /* nsEscCharSetProber_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEscSM.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEscSM.cpp deleted file mode 100644 index eed1b7c..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsEscSM.cpp +++ /dev/null @@ -1,263 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -#include "nsCodingStateMachine.h" - -static const PRUint32 HZ_cls[ 256 / 8 ] = { -PCK4BITS(1,0,0,0,0,0,0,0), // 00 - 07 -PCK4BITS(0,0,0,0,0,0,0,0), // 08 - 0f -PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17 -PCK4BITS(0,0,0,1,0,0,0,0), // 18 - 1f -PCK4BITS(0,0,0,0,0,0,0,0), // 20 - 27 -PCK4BITS(0,0,0,0,0,0,0,0), // 28 - 2f -PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37 -PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f -PCK4BITS(0,0,0,0,0,0,0,0), // 40 - 47 -PCK4BITS(0,0,0,0,0,0,0,0), // 48 - 4f -PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57 -PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f -PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67 -PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f -PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77 -PCK4BITS(0,0,0,4,0,5,2,0), // 78 - 7f -PCK4BITS(1,1,1,1,1,1,1,1), // 80 - 87 -PCK4BITS(1,1,1,1,1,1,1,1), // 88 - 8f -PCK4BITS(1,1,1,1,1,1,1,1), // 90 - 97 -PCK4BITS(1,1,1,1,1,1,1,1), // 98 - 9f -PCK4BITS(1,1,1,1,1,1,1,1), // a0 - a7 -PCK4BITS(1,1,1,1,1,1,1,1), // a8 - af -PCK4BITS(1,1,1,1,1,1,1,1), // b0 - b7 -PCK4BITS(1,1,1,1,1,1,1,1), // b8 - bf -PCK4BITS(1,1,1,1,1,1,1,1), // c0 - c7 -PCK4BITS(1,1,1,1,1,1,1,1), // c8 - cf -PCK4BITS(1,1,1,1,1,1,1,1), // d0 - d7 -PCK4BITS(1,1,1,1,1,1,1,1), // d8 - df -PCK4BITS(1,1,1,1,1,1,1,1), // e0 - e7 -PCK4BITS(1,1,1,1,1,1,1,1), // e8 - ef -PCK4BITS(1,1,1,1,1,1,1,1), // f0 - f7 -PCK4BITS(1,1,1,1,1,1,1,1) // f8 - ff -}; - - -static const PRUint32 HZ_st [ 6] = { -PCK4BITS(eStart,eError, 3,eStart,eStart,eStart,eError,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eError,eError,eStart,eStart, 4,eError),//10-17 -PCK4BITS( 5,eError, 6,eError, 5, 5, 4,eError),//18-1f -PCK4BITS( 4,eError, 4, 4, 4,eError, 4,eError),//20-27 -PCK4BITS( 4,eItsMe,eStart,eStart,eStart,eStart,eStart,eStart) //28-2f -}; - -static const PRUint32 HZCharLenTable[] = {0, 0, 0, 0, 0, 0}; - -const SMModel HZSMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, HZ_cls }, - 6, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, HZ_st }, - HZCharLenTable, - "HZ-GB-2312", -}; - - -static const PRUint32 ISO2022CN_cls [ 256 / 8 ] = { -PCK4BITS(2,0,0,0,0,0,0,0), // 00 - 07 -PCK4BITS(0,0,0,0,0,0,0,0), // 08 - 0f -PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17 -PCK4BITS(0,0,0,1,0,0,0,0), // 18 - 1f -PCK4BITS(0,0,0,0,0,0,0,0), // 20 - 27 -PCK4BITS(0,3,0,0,0,0,0,0), // 28 - 2f -PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37 -PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f -PCK4BITS(0,0,0,4,0,0,0,0), // 40 - 47 -PCK4BITS(0,0,0,0,0,0,0,0), // 48 - 4f -PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57 -PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f -PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67 -PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f -PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77 -PCK4BITS(0,0,0,0,0,0,0,0), // 78 - 7f -PCK4BITS(2,2,2,2,2,2,2,2), // 80 - 87 -PCK4BITS(2,2,2,2,2,2,2,2), // 88 - 8f -PCK4BITS(2,2,2,2,2,2,2,2), // 90 - 97 -PCK4BITS(2,2,2,2,2,2,2,2), // 98 - 9f -PCK4BITS(2,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 -PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef -PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 -PCK4BITS(2,2,2,2,2,2,2,2) // f8 - ff -}; - - -static const PRUint32 ISO2022CN_st [ 8] = { -PCK4BITS(eStart, 3,eError,eStart,eStart,eStart,eStart,eStart),//00-07 -PCK4BITS(eStart,eError,eError,eError,eError,eError,eError,eError),//08-0f -PCK4BITS(eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe),//10-17 -PCK4BITS(eItsMe,eItsMe,eItsMe,eError,eError,eError, 4,eError),//18-1f -PCK4BITS(eError,eError,eError,eItsMe,eError,eError,eError,eError),//20-27 -PCK4BITS( 5, 6,eError,eError,eError,eError,eError,eError),//28-2f -PCK4BITS(eError,eError,eError,eItsMe,eError,eError,eError,eError),//30-37 -PCK4BITS(eError,eError,eError,eError,eError,eItsMe,eError,eStart) //38-3f -}; - -static const PRUint32 ISO2022CNCharLenTable[] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; - -const SMModel ISO2022CNSMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, ISO2022CN_cls }, - 9, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, ISO2022CN_st }, - ISO2022CNCharLenTable, - "ISO-2022-CN", -}; - -static const PRUint32 ISO2022JP_cls [ 256 / 8 ] = { -PCK4BITS(2,0,0,0,0,0,0,0), // 00 - 07 -PCK4BITS(0,0,0,0,0,0,2,2), // 08 - 0f -PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17 -PCK4BITS(0,0,0,1,0,0,0,0), // 18 - 1f -PCK4BITS(0,0,0,0,7,0,0,0), // 20 - 27 -PCK4BITS(3,0,0,0,0,0,0,0), // 28 - 2f -PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37 -PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f -PCK4BITS(6,0,4,0,8,0,0,0), // 40 - 47 -PCK4BITS(0,9,5,0,0,0,0,0), // 48 - 4f -PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57 -PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f -PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67 -PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f -PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77 -PCK4BITS(0,0,0,0,0,0,0,0), // 78 - 7f -PCK4BITS(2,2,2,2,2,2,2,2), // 80 - 87 -PCK4BITS(2,2,2,2,2,2,2,2), // 88 - 8f -PCK4BITS(2,2,2,2,2,2,2,2), // 90 - 97 -PCK4BITS(2,2,2,2,2,2,2,2), // 98 - 9f -PCK4BITS(2,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 -PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef -PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 -PCK4BITS(2,2,2,2,2,2,2,2) // f8 - ff -}; - - -static const PRUint32 ISO2022JP_st [ 9] = { -PCK4BITS(eStart, 3,eError,eStart,eStart,eStart,eStart,eStart),//00-07 -PCK4BITS(eStart,eStart,eError,eError,eError,eError,eError,eError),//08-0f -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//10-17 -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError),//18-1f -PCK4BITS(eError, 5,eError,eError,eError, 4,eError,eError),//20-27 -PCK4BITS(eError,eError,eError, 6,eItsMe,eError,eItsMe,eError),//28-2f -PCK4BITS(eError,eError,eError,eError,eError,eError,eItsMe,eItsMe),//30-37 -PCK4BITS(eError,eError,eError,eItsMe,eError,eError,eError,eError),//38-3f -PCK4BITS(eError,eError,eError,eError,eItsMe,eError,eStart,eStart) //40-47 -}; - -static const PRUint32 ISO2022JPCharLenTable[] = {0, 0, 0, 0, 0, 0, 0, 0}; - -const SMModel ISO2022JPSMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, ISO2022JP_cls }, - 10, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, ISO2022JP_st }, - ISO2022JPCharLenTable, - "ISO-2022-JP", -}; - -static const PRUint32 ISO2022KR_cls [ 256 / 8 ] = { -PCK4BITS(2,0,0,0,0,0,0,0), // 00 - 07 -PCK4BITS(0,0,0,0,0,0,0,0), // 08 - 0f -PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17 -PCK4BITS(0,0,0,1,0,0,0,0), // 18 - 1f -PCK4BITS(0,0,0,0,3,0,0,0), // 20 - 27 -PCK4BITS(0,4,0,0,0,0,0,0), // 28 - 2f -PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37 -PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f -PCK4BITS(0,0,0,5,0,0,0,0), // 40 - 47 -PCK4BITS(0,0,0,0,0,0,0,0), // 48 - 4f -PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57 -PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f -PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67 -PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f -PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77 -PCK4BITS(0,0,0,0,0,0,0,0), // 78 - 7f -PCK4BITS(2,2,2,2,2,2,2,2), // 80 - 87 -PCK4BITS(2,2,2,2,2,2,2,2), // 88 - 8f -PCK4BITS(2,2,2,2,2,2,2,2), // 90 - 97 -PCK4BITS(2,2,2,2,2,2,2,2), // 98 - 9f -PCK4BITS(2,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 -PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef -PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 -PCK4BITS(2,2,2,2,2,2,2,2) // f8 - ff -}; - - -static const PRUint32 ISO2022KR_st [ 5] = { -PCK4BITS(eStart, 3,eError,eStart,eStart,eStart,eError,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eError,eError,eError, 4,eError,eError),//10-17 -PCK4BITS(eError,eError,eError,eError, 5,eError,eError,eError),//18-1f -PCK4BITS(eError,eError,eError,eItsMe,eStart,eStart,eStart,eStart) //20-27 -}; - -static const PRUint32 ISO2022KRCharLenTable[] = {0, 0, 0, 0, 0, 0}; - -const SMModel ISO2022KRSMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, ISO2022KR_cls }, - 6, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, ISO2022KR_st }, - ISO2022KRCharLenTable, - "ISO-2022-KR", -}; - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsGB2312Prober.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsGB2312Prober.cpp deleted file mode 100644 index 7aeee0e..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsGB2312Prober.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -// for S-JIS encoding, obeserve characteristic: -// 1, kana character (or hankaku?) often have hight frequency of appereance -// 2, kana character often exist in group -// 3, certain combination of kana is never used in japanese language - -#include "nsGB2312Prober.h" -#include "nsDebug.h" - -void nsGB18030Prober::Reset(void) -{ - mCodingSM->Reset(); - mState = eDetecting; - mDistributionAnalyser.Reset(mIsPreferredLanguage); - //mContextAnalyser.Reset(); -} - -nsProbingState nsGB18030Prober::HandleData(const char* aBuf, PRUint32 aLen) -{ - NS_ASSERTION(aLen, "HandleData called with empty buffer"); - nsSMState codingState; - - for (PRUint32 i = 0; i < aLen; i++) - { - codingState = mCodingSM->NextState(aBuf[i]); - if (codingState == eItsMe) - { - mState = eFoundIt; - break; - } - if (codingState == eStart) - { - PRUint32 charLen = mCodingSM->GetCurrentCharLen(); - - if (i == 0) - { - mLastChar[1] = aBuf[0]; - mDistributionAnalyser.HandleOneChar(mLastChar, charLen); - } - else - mDistributionAnalyser.HandleOneChar(aBuf+i-1, charLen); - } - } - - mLastChar[0] = aBuf[aLen-1]; - - if (mState == eDetecting) - if (mDistributionAnalyser.GotEnoughData() && GetConfidence() > SHORTCUT_THRESHOLD) - mState = eFoundIt; -// else -// mDistributionAnalyser.HandleData(aBuf, aLen); - - return mState; -} - -float nsGB18030Prober::GetConfidence(void) -{ - float distribCf = mDistributionAnalyser.GetConfidence(); - - return (float)distribCf; -} - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsGB2312Prober.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsGB2312Prober.h deleted file mode 100644 index 4d6cd56..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsGB2312Prober.h +++ /dev/null @@ -1,76 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsGB2312Prober_h__ -#define nsGB2312Prober_h__ - -#include "nsCharSetProber.h" -#include "nsCodingStateMachine.h" -#include "CharDistribution.h" - -// We use gb18030 to replace gb2312, because 18030 is a superset. - -class nsGB18030Prober: public nsCharSetProber { -public: - nsGB18030Prober(PRBool aIsPreferredLanguage) - :mIsPreferredLanguage(aIsPreferredLanguage) - {mCodingSM = new nsCodingStateMachine(&GB18030SMModel); - Reset();} - virtual ~nsGB18030Prober(void){delete mCodingSM;} - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName() {return "GB18030";} - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void); - void SetOpion() {} - -protected: - void GetDistribution(PRUint32 aCharLen, const char* aStr); - - nsCodingStateMachine* mCodingSM; - nsProbingState mState; - - //GB2312ContextAnalysis mContextAnalyser; - GB2312DistributionAnalysis mDistributionAnalyser; - char mLastChar[2]; - PRBool mIsPreferredLanguage; - -}; - - -#endif /* nsGB2312Prober_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsHebrewProber.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsHebrewProber.cpp deleted file mode 100644 index c503617..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsHebrewProber.cpp +++ /dev/null @@ -1,194 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Shy Shalom - * Portions created by the Initial Developer are Copyright (C) 2005 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsHebrewProber.h" -#include - -// windows-1255 / ISO-8859-8 code points of interest -#define FINAL_KAF ('\xea') -#define NORMAL_KAF ('\xeb') -#define FINAL_MEM ('\xed') -#define NORMAL_MEM ('\xee') -#define FINAL_NUN ('\xef') -#define NORMAL_NUN ('\xf0') -#define FINAL_PE ('\xf3') -#define NORMAL_PE ('\xf4') -#define FINAL_TSADI ('\xf5') -#define NORMAL_TSADI ('\xf6') - -// Minimum Visual vs Logical final letter score difference. -// If the difference is below this, don't rely solely on the final letter score distance. -#define MIN_FINAL_CHAR_DISTANCE (5) - -// Minimum Visual vs Logical model score difference. -// If the difference is below this, don't rely at all on the model score distance. -#define MIN_MODEL_DISTANCE (0.01) - -#define VISUAL_HEBREW_NAME ("ISO-8859-8") -#define LOGICAL_HEBREW_NAME ("WINDOWS-1255") - -PRBool nsHebrewProber::isFinal(char c) -{ - return ((c == FINAL_KAF) || (c == FINAL_MEM) || (c == FINAL_NUN) || (c == FINAL_PE) || (c == FINAL_TSADI)); -} - -PRBool nsHebrewProber::isNonFinal(char c) -{ - return ((c == NORMAL_KAF) || (c == NORMAL_MEM) || (c == NORMAL_NUN) || (c == NORMAL_PE)); - // The normal Tsadi is not a good Non-Final letter due to words like - // 'lechotet' (to chat) containing an apostrophe after the tsadi. This - // apostrophe is converted to a space in FilterWithoutEnglishLetters causing - // the Non-Final tsadi to appear at an end of a word even though this is not - // the case in the original text. - // The letters Pe and Kaf rarely display a related behavior of not being a - // good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak' for - // example legally end with a Non-Final Pe or Kaf. However, the benefit of - // these letters as Non-Final letters outweighs the damage since these words - // are quite rare. -} - -/** HandleData - * Final letter analysis for logical-visual decision. - * Look for evidence that the received buffer is either logical Hebrew or - * visual Hebrew. - * The following cases are checked: - * 1) A word longer than 1 letter, ending with a final letter. This is an - * indication that the text is laid out "naturally" since the final letter - * really appears at the end. +1 for logical score. - * 2) A word longer than 1 letter, ending with a Non-Final letter. In normal - * Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi, should not end with - * the Non-Final form of that letter. Exceptions to this rule are mentioned - * above in isNonFinal(). This is an indication that the text is laid out - * backwards. +1 for visual score - * 3) A word longer than 1 letter, starting with a final letter. Final letters - * should not appear at the beginning of a word. This is an indication that - * the text is laid out backwards. +1 for visual score. - * - * The visual score and logical score are accumulated throughout the text and - * are finally checked against each other in GetCharSetName(). - * No checking for final letters in the middle of words is done since that case - * is not an indication for either Logical or Visual text. - * - * The input buffer should not contain any white spaces that are not (' ') - * or any low-ascii punctuation marks. - */ -nsProbingState nsHebrewProber::HandleData(const char* aBuf, PRUint32 aLen) -{ - // Both model probers say it's not them. No reason to continue. - if (GetState() == eNotMe) - return eNotMe; - - const char *curPtr, *endPtr = aBuf+aLen; - char cur; - - for (curPtr = (char*)aBuf; curPtr < endPtr; ++curPtr) - { - cur = *curPtr; - if (cur == ' ') // We stand on a space - a word just ended - { - if (mBeforePrev != ' ') // *(curPtr-2) was not a space so prev is not a 1 letter word - { - if (isFinal(mPrev)) // case (1) [-2:not space][-1:final letter][cur:space] - ++mFinalCharLogicalScore; - else if (isNonFinal(mPrev)) // case (2) [-2:not space][-1:Non-Final letter][cur:space] - ++mFinalCharVisualScore; - } - } - else // Not standing on a space - { - if ((mBeforePrev == ' ') && (isFinal(mPrev)) && (cur != ' ')) // case (3) [-2:space][-1:final letter][cur:not space] - ++mFinalCharVisualScore; - } - mBeforePrev = mPrev; - mPrev = cur; - } - - // Forever detecting, till the end or until both model probers return eNotMe (handled above). - return eDetecting; -} - -// Make the decision: is it Logical or Visual? -const char* nsHebrewProber::GetCharSetName() -{ - // If the final letter score distance is dominant enough, rely on it. - PRInt32 finalsub = mFinalCharLogicalScore - mFinalCharVisualScore; - if (finalsub >= MIN_FINAL_CHAR_DISTANCE) - return LOGICAL_HEBREW_NAME; - if (finalsub <= -(MIN_FINAL_CHAR_DISTANCE)) - return VISUAL_HEBREW_NAME; - - // It's not dominant enough, try to rely on the model scores instead. - float modelsub = mLogicalProb->GetConfidence() - mVisualProb->GetConfidence(); - if (modelsub > MIN_MODEL_DISTANCE) - return LOGICAL_HEBREW_NAME; - if (modelsub < -(MIN_MODEL_DISTANCE)) - return VISUAL_HEBREW_NAME; - - // Still no good, back to final letter distance, maybe it'll save the day. - if (finalsub < 0) - return VISUAL_HEBREW_NAME; - - // (finalsub > 0 - Logical) or (don't know what to do) default to Logical. - return LOGICAL_HEBREW_NAME; -} - - -void nsHebrewProber::Reset(void) -{ - mFinalCharLogicalScore = 0; - mFinalCharVisualScore = 0; - - // mPrev and mBeforePrev are initialized to space in order to simulate a word - // delimiter at the beginning of the data - mPrev = ' '; - mBeforePrev = ' '; -} - -nsProbingState nsHebrewProber::GetState(void) -{ - // Remain active as long as any of the model probers are active. - if ((mLogicalProb->GetState() == eNotMe) && (mVisualProb->GetState() == eNotMe)) - return eNotMe; - return eDetecting; -} - -#ifdef DEBUG_chardet -void nsHebrewProber::DumpStatus() -{ - printf(" HEB: %d - %d [Logical-Visual score]\r\n", mFinalCharLogicalScore, mFinalCharVisualScore); -} -#endif diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsHebrewProber.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsHebrewProber.h deleted file mode 100644 index eedfed4..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsHebrewProber.h +++ /dev/null @@ -1,176 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Shy Shalom - * Portions created by the Initial Developer are Copyright (C) 2005 - * the Initial Developer: All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsHebrewProber_h__ -#define nsHebrewProber_h__ - -#include "nsSBCharSetProber.h" - -// This prober doesn't actually recognize a language or a charset. -// It is a helper prober for the use of the Hebrew model probers -class nsHebrewProber: public nsCharSetProber -{ -public: - nsHebrewProber(void) :mLogicalProb(0), mVisualProb(0) { Reset(); } - - virtual ~nsHebrewProber(void) {} - virtual nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - virtual const char* GetCharSetName(); - virtual void Reset(void); - - virtual nsProbingState GetState(void); - - virtual float GetConfidence(void) { return (float)0.0; } - virtual void SetOpion() {} - - void SetModelProbers(nsCharSetProber *logicalPrb, nsCharSetProber *visualPrb) - { mLogicalProb = logicalPrb; mVisualProb = visualPrb; } - -#ifdef DEBUG_chardet - virtual void DumpStatus(); -#endif - -protected: - static PRBool isFinal(char c); - static PRBool isNonFinal(char c); - - PRInt32 mFinalCharLogicalScore, mFinalCharVisualScore; - - // The two last characters seen in the previous buffer. - char mPrev, mBeforePrev; - - // These probers are owned by the group prober. - nsCharSetProber *mLogicalProb, *mVisualProb; -}; - -/** - * ** General ideas of the Hebrew charset recognition ** - * - * Four main charsets exist in Hebrew: - * "ISO-8859-8" - Visual Hebrew - * "windows-1255" - Logical Hebrew - * "ISO-8859-8-I" - Logical Hebrew - * "x-mac-hebrew" - ?? Logical Hebrew ?? - * - * Both "ISO" charsets use a completely identical set of code points, whereas - * "windows-1255" and "x-mac-hebrew" are two different proper supersets of - * these code points. windows-1255 defines additional characters in the range - * 0x80-0x9F as some misc punctuation marks as well as some Hebrew-specific - * diacritics and additional 'Yiddish' ligature letters in the range 0xc0-0xd6. - * x-mac-hebrew defines similar additional code points but with a different - * mapping. - * - * As far as an average Hebrew text with no diacritics is concerned, all four - * charsets are identical with respect to code points. Meaning that for the - * main Hebrew alphabet, all four map the same values to all 27 Hebrew letters - * (including final letters). - * - * The dominant difference between these charsets is their directionality. - * "Visual" directionality means that the text is ordered as if the renderer is - * not aware of a BIDI rendering algorithm. The renderer sees the text and - * draws it from left to right. The text itself when ordered naturally is read - * backwards. A buffer of Visual Hebrew generally looks like so: - * "[last word of first line spelled backwards] [whole line ordered backwards - * and spelled backwards] [first word of first line spelled backwards] - * [end of line] [last word of second line] ... etc' " - * adding punctuation marks, numbers and English text to visual text is - * naturally also "visual" and from left to right. - * - * "Logical" directionality means the text is ordered "naturally" according to - * the order it is read. It is the responsibility of the renderer to display - * the text from right to left. A BIDI algorithm is used to place general - * punctuation marks, numbers and English text in the text. - * - * Texts in x-mac-hebrew are almost impossible to find on the Internet. From - * what little evidence I could find, it seems that its general directionality - * is Logical. - * - * To sum up all of the above, the Hebrew probing mechanism knows about two - * charsets: - * Visual Hebrew - "ISO-8859-8" - backwards text - Words and sentences are - * backwards while line order is natural. For charset recognition purposes - * the line order is unimportant (In fact, for this implementation, even - * word order is unimportant). - * Logical Hebrew - "windows-1255" - normal, naturally ordered text. - * - * "ISO-8859-8-I" is a subset of windows-1255 and doesn't need to be - * specifically identified. - * "x-mac-hebrew" is also identified as windows-1255. A text in x-mac-hebrew - * that contain special punctuation marks or diacritics is displayed with - * some unconverted characters showing as question marks. This problem might - * be corrected using another model prober for x-mac-hebrew. Due to the fact - * that x-mac-hebrew texts are so rare, writing another model prober isn't - * worth the effort and performance hit. - * - * *** The Prober *** - * - * The prober is divided between two nsSBCharSetProbers and an nsHebrewProber, - * all of which are managed, created, fed data, inquired and deleted by the - * nsSBCSGroupProber. The two nsSBCharSetProbers identify that the text is in - * fact some kind of Hebrew, Logical or Visual. The final decision about which - * one is it is made by the nsHebrewProber by combining final-letter scores - * with the scores of the two nsSBCharSetProbers to produce a final answer. - * - * The nsSBCSGroupProber is responsible for stripping the original text of HTML - * tags, English characters, numbers, low-ASCII punctuation characters, spaces - * and new lines. It reduces any sequence of such characters to a single space. - * The buffer fed to each prober in the SBCS group prober is pure text in - * high-ASCII. - * The two nsSBCharSetProbers (model probers) share the same language model: - * Win1255Model. - * The first nsSBCharSetProber uses the model normally as any other - * nsSBCharSetProber does, to recognize windows-1255, upon which this model was - * built. The second nsSBCharSetProber is told to make the pair-of-letter - * lookup in the language model backwards. This in practice exactly simulates - * a visual Hebrew model using the windows-1255 logical Hebrew model. - * - * The nsHebrewProber is not using any language model. All it does is look for - * final-letter evidence suggesting the text is either logical Hebrew or visual - * Hebrew. Disjointed from the model probers, the results of the nsHebrewProber - * alone are meaningless. nsHebrewProber always returns 0.00 as confidence - * since it never identifies a charset by itself. Instead, the pointer to the - * nsHebrewProber is passed to the model probers as a helper "Name Prober". - * When the Group prober receives a positive identification from any prober, - * it asks for the name of the charset identified. If the prober queried is a - * Hebrew model prober, the model prober forwards the call to the - * nsHebrewProber to make the final decision. In the nsHebrewProber, the - * decision is made according to the final-letters scores maintained and Both - * model probers scores. The answer is returned in the form of the name of the - * charset identified, either "windows-1255" or "ISO-8859-8". - * - */ -#endif /* nsHebrewProber_h__ */ diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsLatin1Prober.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsLatin1Prober.cpp deleted file mode 100644 index 7694ef7..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsLatin1Prober.cpp +++ /dev/null @@ -1,182 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 2001 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shy Shalom - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsLatin1Prober.h" -#include "prmem.h" -#include - -#define UDF 0 // undefined -#define OTH 1 //other -#define ASC 2 // ascii capital letter -#define ASS 3 // ascii small letter -#define ACV 4 // accent capital vowel -#define ACO 5 // accent capital other -#define ASV 6 // accent small vowel -#define ASO 7 // accent small other -#define CLASS_NUM 8 // total classes - -static const unsigned char Latin1_CharToClass[] = -{ - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 00 - 07 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 08 - 0F - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 10 - 17 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 18 - 1F - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 20 - 27 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 28 - 2F - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 30 - 37 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 38 - 3F - OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, // 40 - 47 - ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, // 48 - 4F - ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, // 50 - 57 - ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, // 58 - 5F - OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, // 60 - 67 - ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, // 68 - 6F - ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, // 70 - 77 - ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, // 78 - 7F - OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, // 80 - 87 - OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, // 88 - 8F - UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 90 - 97 - OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, // 98 - 9F - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // A0 - A7 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // A8 - AF - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // B0 - B7 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // B8 - BF - ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, // C0 - C7 - ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, // C8 - CF - ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, // D0 - D7 - ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, // D8 - DF - ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, // E0 - E7 - ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, // E8 - EF - ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, // F0 - F7 - ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, // F8 - FF -}; - - -/* 0 : illegal - 1 : very unlikely - 2 : normal - 3 : very likely -*/ -static const unsigned char Latin1ClassModel[] = -{ -/* UDF OTH ASC ASS ACV ACO ASV ASO */ -/*UDF*/ 0, 0, 0, 0, 0, 0, 0, 0, -/*OTH*/ 0, 3, 3, 3, 3, 3, 3, 3, -/*ASC*/ 0, 3, 3, 3, 3, 3, 3, 3, -/*ASS*/ 0, 3, 3, 3, 1, 1, 3, 3, -/*ACV*/ 0, 3, 3, 3, 1, 2, 1, 2, -/*ACO*/ 0, 3, 3, 3, 3, 3, 3, 3, -/*ASV*/ 0, 3, 1, 3, 1, 1, 1, 3, -/*ASO*/ 0, 3, 1, 3, 1, 1, 3, 3, -}; - -void nsLatin1Prober::Reset(void) -{ - mState = eDetecting; - mLastCharClass = OTH; - for (int i = 0; i < FREQ_CAT_NUM; i++) - mFreqCounter[i] = 0; -} - - -nsProbingState nsLatin1Prober::HandleData(const char* aBuf, PRUint32 aLen) -{ - char *newBuf1 = 0; - PRUint32 newLen1 = 0; - - if (!FilterWithEnglishLetters(aBuf, aLen, &newBuf1, newLen1)) { - newBuf1 = (char*)aBuf; - newLen1 = aLen; - } - - unsigned char charClass; - unsigned char freq; - for (PRUint32 i = 0; i < newLen1; i++) - { - charClass = Latin1_CharToClass[(unsigned char)newBuf1[i]]; - freq = Latin1ClassModel[mLastCharClass*CLASS_NUM + charClass]; - if (freq == 0) { - mState = eNotMe; - break; - } - mFreqCounter[freq]++; - mLastCharClass = charClass; - } - - if (newBuf1 != aBuf) - PR_FREEIF(newBuf1); - - return mState; -} - -float nsLatin1Prober::GetConfidence(void) -{ - if (mState == eNotMe) - return 0.01f; - - float confidence; - PRUint32 total = 0; - for (PRInt32 i = 0; i < FREQ_CAT_NUM; i++) - total += mFreqCounter[i]; - - if(!total) - confidence = 0.0f; - else - { - confidence = mFreqCounter[3]*1.0f / total; - confidence -= mFreqCounter[1]*20.0f/total; - } - - if (confidence < 0.0f) - confidence = 0.0f; - - // lower the confidence of latin1 so that other more accurate detector - // can take priority. - confidence *= 0.50f; - - return confidence; -} - -#ifdef DEBUG_chardet -void nsLatin1Prober::DumpStatus() -{ - printf(" Latin1Prober: %1.3f [%s]\r\n", GetConfidence(), GetCharSetName()); -} -#endif - - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsLatin1Prober.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsLatin1Prober.h deleted file mode 100644 index 59118a7..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsLatin1Prober.h +++ /dev/null @@ -1,70 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 2001 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shy Shalom - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsLatin1Prober_h__ -#define nsLatin1Prober_h__ - -#include "nsCharSetProber.h" - -#define FREQ_CAT_NUM 4 - -class nsLatin1Prober: public nsCharSetProber { -public: - nsLatin1Prober(void){Reset();} - virtual ~nsLatin1Prober(void){} - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName() {return "WINDOWS-1252";} - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void); - void SetOpion() {} - -#ifdef DEBUG_chardet - virtual void DumpStatus(); -#endif - -protected: - - nsProbingState mState; - char mLastCharClass; - PRUint32 mFreqCounter[FREQ_CAT_NUM]; -}; - - -#endif /* nsLatin1Prober_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsMBCSGroupProber.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsMBCSGroupProber.cpp deleted file mode 100644 index f161165..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsMBCSGroupProber.cpp +++ /dev/null @@ -1,230 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 2001 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shy Shalom - * Proofpoint, Inc. - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -#include - -#include "nsMBCSGroupProber.h" -#include "nsUniversalDetector.h" - -#if defined(DEBUG_chardet) || defined(DEBUG_jgmyers) -const char *ProberName[] = -{ - "UTF8", - "SJIS", - "EUCJP", - "GB18030", - "EUCKR", - "Big5", - "EUCTW", -}; - -#endif - -nsMBCSGroupProber::nsMBCSGroupProber(PRUint32 aLanguageFilter) -{ - for (PRUint32 i = 0; i < NUM_OF_PROBERS; i++) - mProbers[i] = nsnull; - - mProbers[0] = new nsUTF8Prober(); - if (aLanguageFilter & NS_FILTER_JAPANESE) - { - mProbers[1] = new nsSJISProber(aLanguageFilter == NS_FILTER_JAPANESE); - mProbers[2] = new nsEUCJPProber(aLanguageFilter == NS_FILTER_JAPANESE); - } - if (aLanguageFilter & NS_FILTER_CHINESE_SIMPLIFIED) - mProbers[3] = new nsGB18030Prober(aLanguageFilter == NS_FILTER_CHINESE_SIMPLIFIED); - if (aLanguageFilter & NS_FILTER_KOREAN) - mProbers[4] = new nsEUCKRProber(aLanguageFilter == NS_FILTER_KOREAN); - if (aLanguageFilter & NS_FILTER_CHINESE_TRADITIONAL) - { - mProbers[5] = new nsBig5Prober(aLanguageFilter == NS_FILTER_CHINESE_TRADITIONAL); - mProbers[6] = new nsEUCTWProber(aLanguageFilter == NS_FILTER_CHINESE_TRADITIONAL); - } - Reset(); -} - -nsMBCSGroupProber::~nsMBCSGroupProber() -{ - for (PRUint32 i = 0; i < NUM_OF_PROBERS; i++) - { - delete mProbers[i]; - } -} - -const char* nsMBCSGroupProber::GetCharSetName() -{ - if (mBestGuess == -1) - { - GetConfidence(); - if (mBestGuess == -1) - mBestGuess = 0; - } - return mProbers[mBestGuess]->GetCharSetName(); -} - -void nsMBCSGroupProber::Reset(void) -{ - mActiveNum = 0; - for (PRUint32 i = 0; i < NUM_OF_PROBERS; i++) - { - if (mProbers[i]) - { - mProbers[i]->Reset(); - mIsActive[i] = PR_TRUE; - ++mActiveNum; - } - else - mIsActive[i] = PR_FALSE; - } - mBestGuess = -1; - mState = eDetecting; - mKeepNext = 0; -} - -nsProbingState nsMBCSGroupProber::HandleData(const char* aBuf, PRUint32 aLen) -{ - nsProbingState st; - PRUint32 start = 0; - PRUint32 keepNext = mKeepNext; - - //do filtering to reduce load to probers - for (PRUint32 pos = 0; pos < aLen; ++pos) - { - if (aBuf[pos] & 0x80) - { - if (!keepNext) - start = pos; - keepNext = 2; - } - else if (keepNext) - { - if (--keepNext == 0) - { - for (PRUint32 i = 0; i < NUM_OF_PROBERS; i++) - { - if (!mIsActive[i]) - continue; - st = mProbers[i]->HandleData(aBuf + start, pos + 1 - start); - if (st == eFoundIt) - { - mBestGuess = i; - mState = eFoundIt; - return mState; - } - } - } - } - } - - if (keepNext) { - for (PRUint32 i = 0; i < NUM_OF_PROBERS; i++) - { - if (!mIsActive[i]) - continue; - st = mProbers[i]->HandleData(aBuf + start, aLen - start); - if (st == eFoundIt) - { - mBestGuess = i; - mState = eFoundIt; - return mState; - } - } - } - mKeepNext = keepNext; - - return mState; -} - -float nsMBCSGroupProber::GetConfidence(void) -{ - PRUint32 i; - float bestConf = 0.0, cf; - - switch (mState) - { - case eFoundIt: - return (float)0.99; - case eNotMe: - return (float)0.01; - default: - for (i = 0; i < NUM_OF_PROBERS; i++) - { - if (!mIsActive[i]) - continue; - cf = mProbers[i]->GetConfidence(); - if (bestConf < cf) - { - bestConf = cf; - mBestGuess = i; - } - } - } - return bestConf; -} - -#ifdef DEBUG_chardet -void nsMBCSGroupProber::DumpStatus() -{ - PRUint32 i; - float cf; - - GetConfidence(); - for (i = 0; i < NUM_OF_PROBERS; i++) - { - if (!mIsActive[i]) - printf(" MBCS inactive: [%s] (confidence is too low).\r\n", ProberName[i]); - else - { - cf = mProbers[i]->GetConfidence(); - printf(" MBCS %1.3f: [%s]\r\n", cf, ProberName[i]); - } - } -} -#endif - -#ifdef DEBUG_jgmyers -void nsMBCSGroupProber::GetDetectorState(nsUniversalDetector::DetectorState (&states)[nsUniversalDetector::NumDetectors], PRUint32 &offset) -{ - for (PRUint32 i = 0; i < NUM_OF_PROBERS; ++i) { - states[offset].name = ProberName[i]; - states[offset].isActive = mIsActive[i]; - states[offset].confidence = mIsActive[i] ? mProbers[i]->GetConfidence() : 0.0; - ++offset; - } -} -#endif /* DEBUG_jgmyers */ diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsMBCSGroupProber.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsMBCSGroupProber.h deleted file mode 100644 index c4e9964..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsMBCSGroupProber.h +++ /dev/null @@ -1,80 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Proofpoint, Inc. - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsMBCSGroupProber_h__ -#define nsMBCSGroupProber_h__ - -#include "nsSJISProber.h" -#include "nsUTF8Prober.h" -#include "nsEUCJPProber.h" -#include "nsGB2312Prober.h" -#include "nsEUCKRProber.h" -#include "nsBig5Prober.h" -#include "nsEUCTWProber.h" - -#define NUM_OF_PROBERS 7 - -class nsMBCSGroupProber: public nsCharSetProber { -public: - nsMBCSGroupProber(PRUint32 aLanguageFilter); - virtual ~nsMBCSGroupProber(); - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName(); - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void); - void SetOpion() {} - -#ifdef DEBUG_chardet - void DumpStatus(); -#endif -#ifdef DEBUG_jgmyers - void GetDetectorState(nsUniversalDetector::DetectorState (&states)[nsUniversalDetector::NumDetectors], PRUint32 &offset); -#endif - -protected: - nsProbingState mState; - nsCharSetProber* mProbers[NUM_OF_PROBERS]; - PRBool mIsActive[NUM_OF_PROBERS]; - PRInt32 mBestGuess; - PRUint32 mActiveNum; - PRUint32 mKeepNext; -}; - -#endif /* nsMBCSGroupProber_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsMBCSSM.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsMBCSSM.cpp deleted file mode 100644 index 02cc5cc..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsMBCSSM.cpp +++ /dev/null @@ -1,513 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -#include "nsCodingStateMachine.h" - -/* -Modification from frank tang's original work: -. 0x00 is allowed as a legal character. Since some web pages contains this char in - text stream. -*/ - -// BIG5 - -static const PRUint32 BIG5_cls [ 256 / 8 ] = { -//PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 //allow 0x00 as legal value -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 -PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 -PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f -PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 -PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f -PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 -PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f -PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 -PCK4BITS(2,2,2,2,2,2,2,1), // 78 - 7f -PCK4BITS(4,4,4,4,4,4,4,4), // 80 - 87 -PCK4BITS(4,4,4,4,4,4,4,4), // 88 - 8f -PCK4BITS(4,4,4,4,4,4,4,4), // 90 - 97 -PCK4BITS(4,4,4,4,4,4,4,4), // 98 - 9f -PCK4BITS(4,3,3,3,3,3,3,3), // a0 - a7 -PCK4BITS(3,3,3,3,3,3,3,3), // a8 - af -PCK4BITS(3,3,3,3,3,3,3,3), // b0 - b7 -PCK4BITS(3,3,3,3,3,3,3,3), // b8 - bf -PCK4BITS(3,3,3,3,3,3,3,3), // c0 - c7 -PCK4BITS(3,3,3,3,3,3,3,3), // c8 - cf -PCK4BITS(3,3,3,3,3,3,3,3), // d0 - d7 -PCK4BITS(3,3,3,3,3,3,3,3), // d8 - df -PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7 -PCK4BITS(3,3,3,3,3,3,3,3), // e8 - ef -PCK4BITS(3,3,3,3,3,3,3,3), // f0 - f7 -PCK4BITS(3,3,3,3,3,3,3,0) // f8 - ff -}; - - -static const PRUint32 BIG5_st [ 3] = { -PCK4BITS(eError,eStart,eStart, 3,eError,eError,eError,eError),//00-07 -PCK4BITS(eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError),//08-0f -PCK4BITS(eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart) //10-17 -}; - -static const PRUint32 Big5CharLenTable[] = {0, 1, 1, 2, 0}; - -SMModel const Big5SMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, BIG5_cls }, - 5, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, BIG5_st }, - Big5CharLenTable, - "Big5", -}; - -static const PRUint32 EUCJP_cls [ 256 / 8 ] = { -//PCK4BITS(5,4,4,4,4,4,4,4), // 00 - 07 -PCK4BITS(4,4,4,4,4,4,4,4), // 00 - 07 -PCK4BITS(4,4,4,4,4,4,5,5), // 08 - 0f -PCK4BITS(4,4,4,4,4,4,4,4), // 10 - 17 -PCK4BITS(4,4,4,5,4,4,4,4), // 18 - 1f -PCK4BITS(4,4,4,4,4,4,4,4), // 20 - 27 -PCK4BITS(4,4,4,4,4,4,4,4), // 28 - 2f -PCK4BITS(4,4,4,4,4,4,4,4), // 30 - 37 -PCK4BITS(4,4,4,4,4,4,4,4), // 38 - 3f -PCK4BITS(4,4,4,4,4,4,4,4), // 40 - 47 -PCK4BITS(4,4,4,4,4,4,4,4), // 48 - 4f -PCK4BITS(4,4,4,4,4,4,4,4), // 50 - 57 -PCK4BITS(4,4,4,4,4,4,4,4), // 58 - 5f -PCK4BITS(4,4,4,4,4,4,4,4), // 60 - 67 -PCK4BITS(4,4,4,4,4,4,4,4), // 68 - 6f -PCK4BITS(4,4,4,4,4,4,4,4), // 70 - 77 -PCK4BITS(4,4,4,4,4,4,4,4), // 78 - 7f -PCK4BITS(5,5,5,5,5,5,5,5), // 80 - 87 -PCK4BITS(5,5,5,5,5,5,1,3), // 88 - 8f -PCK4BITS(5,5,5,5,5,5,5,5), // 90 - 97 -PCK4BITS(5,5,5,5,5,5,5,5), // 98 - 9f -PCK4BITS(5,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(0,0,0,0,0,0,0,0), // e0 - e7 -PCK4BITS(0,0,0,0,0,0,0,0), // e8 - ef -PCK4BITS(0,0,0,0,0,0,0,0), // f0 - f7 -PCK4BITS(0,0,0,0,0,0,0,5) // f8 - ff -}; - - -static const PRUint32 EUCJP_st [ 5] = { -PCK4BITS( 3, 4, 3, 5,eStart,eError,eError,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eStart,eError,eStart,eError,eError,eError),//10-17 -PCK4BITS(eError,eError,eStart,eError,eError,eError, 3,eError),//18-1f -PCK4BITS( 3,eError,eError,eError,eStart,eStart,eStart,eStart) //20-27 -}; - -static const PRUint32 EUCJPCharLenTable[] = {2, 2, 2, 3, 1, 0}; - -const SMModel EUCJPSMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCJP_cls }, - 6, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCJP_st }, - EUCJPCharLenTable, - "EUC-JP", -}; - -static const PRUint32 EUCKR_cls [ 256 / 8 ] = { -//PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 -PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47 -PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f -PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57 -PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f -PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67 -PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f -PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77 -PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f -PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87 -PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f -PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 -PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f -PCK4BITS(0,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,3,3,3), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,3,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 -PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef -PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 -PCK4BITS(2,2,2,2,2,2,2,0) // f8 - ff -}; - - -static const PRUint32 EUCKR_st [ 2] = { -PCK4BITS(eError,eStart, 3,eError,eError,eError,eError,eError),//00-07 -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart) //08-0f -}; - -static const PRUint32 EUCKRCharLenTable[] = {0, 1, 2, 0}; - -const SMModel EUCKRSMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCKR_cls }, - 4, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCKR_st }, - EUCKRCharLenTable, - "EUC-KR", -}; - -static const PRUint32 EUCTW_cls [ 256 / 8 ] = { -//PCK4BITS(0,2,2,2,2,2,2,2), // 00 - 07 -PCK4BITS(2,2,2,2,2,2,2,2), // 00 - 07 -PCK4BITS(2,2,2,2,2,2,0,0), // 08 - 0f -PCK4BITS(2,2,2,2,2,2,2,2), // 10 - 17 -PCK4BITS(2,2,2,0,2,2,2,2), // 18 - 1f -PCK4BITS(2,2,2,2,2,2,2,2), // 20 - 27 -PCK4BITS(2,2,2,2,2,2,2,2), // 28 - 2f -PCK4BITS(2,2,2,2,2,2,2,2), // 30 - 37 -PCK4BITS(2,2,2,2,2,2,2,2), // 38 - 3f -PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 -PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f -PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 -PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f -PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 -PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f -PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 -PCK4BITS(2,2,2,2,2,2,2,2), // 78 - 7f -PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87 -PCK4BITS(0,0,0,0,0,0,6,0), // 88 - 8f -PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 -PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f -PCK4BITS(0,3,4,4,4,4,4,4), // a0 - a7 -PCK4BITS(5,5,1,1,1,1,1,1), // a8 - af -PCK4BITS(1,1,1,1,1,1,1,1), // b0 - b7 -PCK4BITS(1,1,1,1,1,1,1,1), // b8 - bf -PCK4BITS(1,1,3,1,3,3,3,3), // c0 - c7 -PCK4BITS(3,3,3,3,3,3,3,3), // c8 - cf -PCK4BITS(3,3,3,3,3,3,3,3), // d0 - d7 -PCK4BITS(3,3,3,3,3,3,3,3), // d8 - df -PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7 -PCK4BITS(3,3,3,3,3,3,3,3), // e8 - ef -PCK4BITS(3,3,3,3,3,3,3,3), // f0 - f7 -PCK4BITS(3,3,3,3,3,3,3,0) // f8 - ff -}; - - -static const PRUint32 EUCTW_st [ 6] = { -PCK4BITS(eError,eError,eStart, 3, 3, 3, 4,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eError,eError,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eStart,eError),//10-17 -PCK4BITS(eStart,eStart,eStart,eError,eError,eError,eError,eError),//18-1f -PCK4BITS( 5,eError,eError,eError,eStart,eError,eStart,eStart),//20-27 -PCK4BITS(eStart,eError,eStart,eStart,eStart,eStart,eStart,eStart) //28-2f -}; - -static const PRUint32 EUCTWCharLenTable[] = {0, 0, 1, 2, 2, 2, 3}; - -const SMModel EUCTWSMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCTW_cls }, - 7, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCTW_st }, - EUCTWCharLenTable, - "x-euc-tw", -}; - -/* obsolete GB2312 by gb18030 -static PRUint32 GB2312_cls [ 256 / 8 ] = { -//PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 -PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47 -PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f -PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57 -PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f -PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67 -PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f -PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77 -PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f -PCK4BITS(1,0,0,0,0,0,0,0), // 80 - 87 -PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f -PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 -PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f -PCK4BITS(0,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,3,3,3,3,3,3), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 -PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef -PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 -PCK4BITS(2,2,2,2,2,2,2,0) // f8 - ff -}; - - -static PRUint32 GB2312_st [ 2] = { -PCK4BITS(eError,eStart, 3,eError,eError,eError,eError,eError),//00-07 -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart) //08-0f -}; - -static const PRUint32 GB2312CharLenTable[] = {0, 1, 2, 0}; - -SMModel GB2312SMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB2312_cls }, - 4, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB2312_st }, - GB2312CharLenTable, - "GB2312", -}; -*/ - -// the following state machine data was created by perl script in -// intl/chardet/tools. It should be the same as in PSM detector. -static const PRUint32 GB18030_cls [ 256 / 8 ] = { -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(3,3,3,3,3,3,3,3), // 30 - 37 -PCK4BITS(3,3,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 -PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f -PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 -PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f -PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 -PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f -PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 -PCK4BITS(2,2,2,2,2,2,2,4), // 78 - 7f -PCK4BITS(5,6,6,6,6,6,6,6), // 80 - 87 -PCK4BITS(6,6,6,6,6,6,6,6), // 88 - 8f -PCK4BITS(6,6,6,6,6,6,6,6), // 90 - 97 -PCK4BITS(6,6,6,6,6,6,6,6), // 98 - 9f -PCK4BITS(6,6,6,6,6,6,6,6), // a0 - a7 -PCK4BITS(6,6,6,6,6,6,6,6), // a8 - af -PCK4BITS(6,6,6,6,6,6,6,6), // b0 - b7 -PCK4BITS(6,6,6,6,6,6,6,6), // b8 - bf -PCK4BITS(6,6,6,6,6,6,6,6), // c0 - c7 -PCK4BITS(6,6,6,6,6,6,6,6), // c8 - cf -PCK4BITS(6,6,6,6,6,6,6,6), // d0 - d7 -PCK4BITS(6,6,6,6,6,6,6,6), // d8 - df -PCK4BITS(6,6,6,6,6,6,6,6), // e0 - e7 -PCK4BITS(6,6,6,6,6,6,6,6), // e8 - ef -PCK4BITS(6,6,6,6,6,6,6,6), // f0 - f7 -PCK4BITS(6,6,6,6,6,6,6,0) // f8 - ff -}; - - -static const PRUint32 GB18030_st [ 6] = { -PCK4BITS(eError,eStart,eStart,eStart,eStart,eStart, 3,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eError,eError,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart),//10-17 -PCK4BITS( 4,eError,eStart,eStart,eError,eError,eError,eError),//18-1f -PCK4BITS(eError,eError, 5,eError,eError,eError,eItsMe,eError),//20-27 -PCK4BITS(eError,eError,eStart,eStart,eStart,eStart,eStart,eStart) //28-2f -}; - -// To be accurate, the length of class 6 can be either 2 or 4. -// But it is not necessary to discriminate between the two since -// it is used for frequency analysis only, and we are validing -// each code range there as well. So it is safe to set it to be -// 2 here. -static const PRUint32 GB18030CharLenTable[] = {0, 1, 1, 1, 1, 1, 2}; - -const SMModel GB18030SMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB18030_cls }, - 7, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB18030_st }, - GB18030CharLenTable, - "GB18030", -}; - -// sjis - -static const PRUint32 SJIS_cls [ 256 / 8 ] = { -//PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 -PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 -PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f -PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 -PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f -PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 -PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f -PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 -PCK4BITS(2,2,2,2,2,2,2,1), // 78 - 7f -PCK4BITS(3,3,3,3,3,3,3,3), // 80 - 87 -PCK4BITS(3,3,3,3,3,3,3,3), // 88 - 8f -PCK4BITS(3,3,3,3,3,3,3,3), // 90 - 97 -PCK4BITS(3,3,3,3,3,3,3,3), // 98 - 9f -//0xa0 is illegal in sjis encoding, but some pages does -//contain such byte. We need to be more error forgiven. -PCK4BITS(2,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7 -PCK4BITS(3,3,3,3,3,4,4,4), // e8 - ef -PCK4BITS(4,4,4,4,4,4,4,4), // f0 - f7 -PCK4BITS(4,4,4,4,4,0,0,0) // f8 - ff -}; - - -static const PRUint32 SJIS_st [ 3] = { -PCK4BITS(eError,eStart,eStart, 3,eError,eError,eError,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eError,eError,eStart,eStart,eStart,eStart) //10-17 -}; - -static const PRUint32 SJISCharLenTable[] = {0, 1, 1, 2, 0, 0}; - -const SMModel SJISSMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, SJIS_cls }, - 6, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, SJIS_st }, - SJISCharLenTable, - "SHIFT_JIS", -}; - - -static const PRUint32 UTF8_cls [ 256 / 8 ] = { -//PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 //allow 0x00 as a legal value -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 -PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47 -PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f -PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57 -PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f -PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67 -PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f -PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77 -PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f -PCK4BITS(2,2,2,2,3,3,3,3), // 80 - 87 -PCK4BITS(4,4,4,4,4,4,4,4), // 88 - 8f -PCK4BITS(4,4,4,4,4,4,4,4), // 90 - 97 -PCK4BITS(4,4,4,4,4,4,4,4), // 98 - 9f -PCK4BITS(5,5,5,5,5,5,5,5), // a0 - a7 -PCK4BITS(5,5,5,5,5,5,5,5), // a8 - af -PCK4BITS(5,5,5,5,5,5,5,5), // b0 - b7 -PCK4BITS(5,5,5,5,5,5,5,5), // b8 - bf -PCK4BITS(0,0,6,6,6,6,6,6), // c0 - c7 -PCK4BITS(6,6,6,6,6,6,6,6), // c8 - cf -PCK4BITS(6,6,6,6,6,6,6,6), // d0 - d7 -PCK4BITS(6,6,6,6,6,6,6,6), // d8 - df -PCK4BITS(7,8,8,8,8,8,8,8), // e0 - e7 -PCK4BITS(8,8,8,8,8,9,8,8), // e8 - ef -PCK4BITS(10,11,11,11,11,11,11,11), // f0 - f7 -PCK4BITS(12,13,13,13,14,15,0,0) // f8 - ff -}; - - -static const PRUint32 UTF8_st [ 26] = { -PCK4BITS(eError,eStart,eError,eError,eError,eError, 12, 10),//00-07 -PCK4BITS( 9, 11, 8, 7, 6, 5, 4, 3),//08-0f -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//10-17 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//18-1f -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe),//20-27 -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe),//28-2f -PCK4BITS(eError,eError, 5, 5, 5, 5,eError,eError),//30-37 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//38-3f -PCK4BITS(eError,eError,eError, 5, 5, 5,eError,eError),//40-47 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//48-4f -PCK4BITS(eError,eError, 7, 7, 7, 7,eError,eError),//50-57 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//58-5f -PCK4BITS(eError,eError,eError,eError, 7, 7,eError,eError),//60-67 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//68-6f -PCK4BITS(eError,eError, 9, 9, 9, 9,eError,eError),//70-77 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//78-7f -PCK4BITS(eError,eError,eError,eError,eError, 9,eError,eError),//80-87 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//88-8f -PCK4BITS(eError,eError, 12, 12, 12, 12,eError,eError),//90-97 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//98-9f -PCK4BITS(eError,eError,eError,eError,eError, 12,eError,eError),//a0-a7 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//a8-af -PCK4BITS(eError,eError, 12, 12, 12,eError,eError,eError),//b0-b7 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//b8-bf -PCK4BITS(eError,eError,eStart,eStart,eStart,eStart,eError,eError),//c0-c7 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError) //c8-cf -}; - -static const PRUint32 UTF8CharLenTable[] = {0, 1, 0, 0, 0, 0, 2, 3, - 3, 3, 4, 4, 5, 5, 6, 6 }; - -const SMModel UTF8SMModel = { - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, UTF8_cls }, - 16, - {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, UTF8_st }, - UTF8CharLenTable, - "UTF-8", -}; - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsPkgInt.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsPkgInt.h deleted file mode 100644 index 3caa912..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsPkgInt.h +++ /dev/null @@ -1,89 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsPkgInt_h__ -#define nsPkgInt_h__ -#include "nscore.h" - -typedef enum { - eIdxSft4bits = 3, - eIdxSft8bits = 2, - eIdxSft16bits = 1 -} nsIdxSft; - -typedef enum { - eSftMsk4bits = 7, - eSftMsk8bits = 3, - eSftMsk16bits = 1 -} nsSftMsk; - -typedef enum { - eBitSft4bits = 2, - eBitSft8bits = 3, - eBitSft16bits = 4 -} nsBitSft; - -typedef enum { - eUnitMsk4bits = 0x0000000FL, - eUnitMsk8bits = 0x000000FFL, - eUnitMsk16bits = 0x0000FFFFL -} nsUnitMsk; - -typedef struct nsPkgInt { - nsIdxSft idxsft; - nsSftMsk sftmsk; - nsBitSft bitsft; - nsUnitMsk unitmsk; - const PRUint32* const data; -} nsPkgInt; - - -#define PCK16BITS(a,b) ((PRUint32)(((b) << 16) | (a))) - -#define PCK8BITS(a,b,c,d) PCK16BITS( ((PRUint32)(((b) << 8) | (a))), \ - ((PRUint32)(((d) << 8) | (c)))) - -#define PCK4BITS(a,b,c,d,e,f,g,h) PCK8BITS( ((PRUint32)(((b) << 4) | (a))), \ - ((PRUint32)(((d) << 4) | (c))), \ - ((PRUint32)(((f) << 4) | (e))), \ - ((PRUint32)(((h) << 4) | (g))) ) - -#define GETFROMPCK(i, c) \ - (((((c).data)[(i)>>(c).idxsft])>>(((i)&(c).sftmsk)<<(c).bitsft))&(c).unitmsk) - -#endif /* nsPkgInt_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCSGroupProber.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCSGroupProber.cpp deleted file mode 100644 index e5b6ba6..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCSGroupProber.cpp +++ /dev/null @@ -1,240 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 2001 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shy Shalom - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include -#include "prmem.h" - -#include "nsSBCharSetProber.h" -#include "nsSBCSGroupProber.h" - -#include "nsHebrewProber.h" - -nsSBCSGroupProber::nsSBCSGroupProber() -{ - mProbers[0] = new nsSingleByteCharSetProber(&Win1251Model); - mProbers[1] = new nsSingleByteCharSetProber(&Koi8rModel); - mProbers[2] = new nsSingleByteCharSetProber(&Latin5Model); - mProbers[3] = new nsSingleByteCharSetProber(&MacCyrillicModel); - mProbers[4] = new nsSingleByteCharSetProber(&Ibm866Model); - mProbers[5] = new nsSingleByteCharSetProber(&Ibm855Model); - mProbers[6] = new nsSingleByteCharSetProber(&ISO_8859_7greekModel); - mProbers[7] = new nsSingleByteCharSetProber(&WINDOWS_1253greekModel); - mProbers[8] = new nsSingleByteCharSetProber(&Latin5BulgarianModel); - mProbers[9] = new nsSingleByteCharSetProber(&Win1251BulgarianModel); - mProbers[10] = new nsSingleByteCharSetProber(&TIS620ThaiModel); - - nsHebrewProber *hebprober = new nsHebrewProber(); - // Notice: Any change in these indexes - 10,11,12 must be reflected - // in the code below as well. - mProbers[11] = hebprober; - mProbers[12] = new nsSingleByteCharSetProber(&Win1255Model, PR_FALSE, hebprober); // Logical Hebrew - mProbers[13] = new nsSingleByteCharSetProber(&Win1255Model, PR_TRUE, hebprober); // Visual Hebrew - // Tell the Hebrew prober about the logical and visual probers - if (mProbers[11] && mProbers[12] && mProbers[13]) // all are not null - { - hebprober->SetModelProbers(mProbers[12], mProbers[13]); - } - else // One or more is null. avoid any Hebrew probing, null them all - { - for (PRUint32 i = 11; i <= 13; ++i) - { - delete mProbers[i]; - mProbers[i] = 0; - } - } - - mProbers[14] = new nsSingleByteCharSetProber(&Latin2HungarianModel); - mProbers[15] = new nsSingleByteCharSetProber(&Win1250HungarianModel); - mProbers[16] = new nsSingleByteCharSetProber(&WINDOWS_1252frenchModel); - mProbers[17] = new nsSingleByteCharSetProber(&WINDOWS_1252germanModel); - mProbers[18] = new nsSingleByteCharSetProber(&WINDOWS_1252swedishModel); - mProbers[19] = new nsSingleByteCharSetProber(&ISO_8859_9turkishModel); - mProbers[20] = new nsSingleByteCharSetProber(&WINDOWS_1252finnishModel); - mProbers[21] = new nsSingleByteCharSetProber(&windows_1252spanishModel); - mProbers[22] = new nsSingleByteCharSetProber(&iso_8859_2czechModel); - mProbers[23] = new nsSingleByteCharSetProber(&iso_8859_2polishModel); - - Reset(); -} - -nsSBCSGroupProber::~nsSBCSGroupProber() -{ - for (PRUint32 i = 0; i < NUM_OF_SBCS_PROBERS; i++) - { - delete mProbers[i]; - } -} - - -const char* nsSBCSGroupProber::GetCharSetName() -{ - //if we have no answer yet - if (mBestGuess == -1) - { - GetConfidence(); - //no charset seems positive - if (mBestGuess == -1) - //we will use default. - mBestGuess = 0; - } - return mProbers[mBestGuess]->GetCharSetName(); -} - -void nsSBCSGroupProber::Reset(void) -{ - mActiveNum = 0; - for (PRUint32 i = 0; i < NUM_OF_SBCS_PROBERS; i++) - { - if (mProbers[i]) // not null - { - mProbers[i]->Reset(); - mIsActive[i] = PR_TRUE; - ++mActiveNum; - } - else - mIsActive[i] = PR_FALSE; - } - mBestGuess = -1; - mState = eDetecting; -} - - -nsProbingState nsSBCSGroupProber::HandleData(const char* aBuf, PRUint32 aLen) -{ - nsProbingState st; - PRUint32 i; - char *newBuf1 = 0; - PRUint32 newLen1 = 0; - char *newBuf2 = 0; - PRUint32 newLen2 = 0; - - if (!FilterWithoutEnglishLetters(aBuf, aLen, &newBuf1, newLen1)) - goto done; - if (!FilterWithEnglishLetters(aBuf, aLen, &newBuf2, newLen2)) - goto done; - - if (newLen1 == 0) - goto done; // Nothing to see here, move on. - - for (i = 0; i < NUM_OF_SBCS_PROBERS; i++) - { - if (!mIsActive[i]) - continue; - if (mProbers[i]->KeepEnglishLetters()) { - st = mProbers[i]->HandleData(newBuf2, newLen2); - } - else - { - st = mProbers[i]->HandleData(newBuf1, newLen1); - } - if (st == eFoundIt) - { - mBestGuess = i; - mState = eFoundIt; - break; - } - else if (st == eNotMe) - { - mIsActive[i] = PR_FALSE; - mActiveNum--; - if (mActiveNum <= 0) - { - mState = eNotMe; - break; - } - } - } - -done: - PR_FREEIF(newBuf1); - PR_FREEIF(newBuf2); - - return mState; -} - -float nsSBCSGroupProber::GetConfidence(void) -{ - PRUint32 i; - float bestConf = 0.0, cf; - - switch (mState) - { - case eFoundIt: - return (float)0.99; //sure yes - case eNotMe: - return (float)0.01; //sure no - default: - for (i = 0; i < NUM_OF_SBCS_PROBERS; i++) - { - if (!mIsActive[i]) - continue; - cf = mProbers[i]->GetConfidence(); - if (bestConf < cf) - { - bestConf = cf; - mBestGuess = i; - } - } - } - return bestConf; -} - -#ifdef DEBUG_chardet -void nsSBCSGroupProber::DumpStatus() -{ - PRUint32 i; - float cf; - - cf = GetConfidence(); - printf(" SBCS Group Prober --------begin status \r\n"); - for (i = 0; i < NUM_OF_SBCS_PROBERS; i++) - { - if (!mIsActive[i]) - printf(" inactive: [%s] [%s](i.e. confidence is too low).\r\n", - mProbers[i]? mProbers[i]->GetCharSetName() : "UNSET", - mProbers[i]? mProbers[i]->GetLangName() : "UNSET" - ); - else - mProbers[i]->DumpStatus(); - } - printf(" SBCS Group found best match [%s] [%s] confidence %f.\r\n", - mProbers[mBestGuess]->GetCharSetName(), - mProbers[mBestGuess]->GetLangName(), - cf); -} -#endif diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCSGroupProber.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCSGroupProber.h deleted file mode 100644 index 575e93f..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCSGroupProber.h +++ /dev/null @@ -1,70 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 2001 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shy Shalom - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsSBCSGroupProber_h__ -#define nsSBCSGroupProber_h__ - - -#define NUM_OF_SBCS_PROBERS 24 - -class nsCharSetProber; -class nsSBCSGroupProber: public nsCharSetProber { -public: - nsSBCSGroupProber(); - virtual ~nsSBCSGroupProber(); - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName(); - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void); - void SetOpion() {} - -#ifdef DEBUG_chardet - void DumpStatus(); -#endif - -protected: - nsProbingState mState; - nsCharSetProber* mProbers[NUM_OF_SBCS_PROBERS]; - PRBool mIsActive[NUM_OF_SBCS_PROBERS]; - PRInt32 mBestGuess; - PRUint32 mActiveNum; -}; - -#endif /* nsSBCSGroupProber_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCharSetProber.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCharSetProber.cpp deleted file mode 100644 index 7ef8004..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCharSetProber.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 2001 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shy Shalom - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -#include -#include "nsSBCharSetProber.h" - -nsProbingState nsSingleByteCharSetProber::HandleData(const char* aBuf, PRUint32 aLen) -{ - unsigned char order; - - for (PRUint32 i = 0; i < aLen; i++) - { - // Order is in [1-64] but we want 0-63 here. - order = mModel->charToOrderMap[(unsigned char)aBuf[i]] -1; - - if (order < SYMBOL_CAT_ORDER) - mTotalChar++; - if (order < SAMPLE_SIZE) - { - mFreqChar++; - - if (mLastOrder < SAMPLE_SIZE) - { - mTotalSeqs++; - if (!mReversed) - ++(mSeqCounters[mModel->precedenceMatrix[mLastOrder*SAMPLE_SIZE+order]]); - else // reverse the order of the letters in the lookup - ++(mSeqCounters[mModel->precedenceMatrix[order*SAMPLE_SIZE+mLastOrder]]); - } - } - mLastOrder = order; - } - - if (mState == eDetecting) - if (mTotalSeqs > SB_ENOUGH_REL_THRESHOLD) - { - float cf = GetConfidence(); - if (cf > POSITIVE_SHORTCUT_THRESHOLD) - mState = eFoundIt; - else if (cf < NEGATIVE_SHORTCUT_THRESHOLD) - mState = eNotMe; - } - - return mState; -} - -void nsSingleByteCharSetProber::Reset(void) -{ - mState = eDetecting; - mLastOrder = 255; - for (PRUint32 i = 0; i < NUMBER_OF_SEQ_CAT; i++) - mSeqCounters[i] = 0; - mTotalSeqs = 0; - mTotalChar = 0; - mFreqChar = 0; -} - -//#define NEGATIVE_APPROACH 1 - -float nsSingleByteCharSetProber::GetConfidence(void) -{ -#ifdef NEGATIVE_APPROACH - if (mTotalSeqs > 0) - if (mTotalSeqs > mSeqCounters[NEGATIVE_CAT]*10 ) - return ((float)(mTotalSeqs - mSeqCounters[NEGATIVE_CAT]*10))/mTotalSeqs * mFreqChar / mTotalChar; - return (float)0.01; -#else //POSITIVE_APPROACH - float r; - - if (mTotalSeqs > 0) { - r = ((float)1.0) * mSeqCounters[POSITIVE_CAT] / mTotalSeqs / mModel->mTypicalPositiveRatio; - r = r*mFreqChar/mTotalChar; - if (r >= (float)1.00) - r = (float)0.99; - return r; - } - return (float)0.01; -#endif -} - -const char* nsSingleByteCharSetProber::GetCharSetName() -{ - if (!mNameProber) - return mModel->charsetName; - return mNameProber->GetCharSetName(); -} - -#ifdef DEBUG_chardet -void nsSingleByteCharSetProber::DumpStatus() -{ - printf(" SBCS: %1.3f [%s] [%s]\r\n", GetConfidence(), mModel->langName, GetCharSetName()); -} -#endif diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCharSetProber.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCharSetProber.h deleted file mode 100644 index da943d2..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSBCharSetProber.h +++ /dev/null @@ -1,129 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 2001 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shy Shalom - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -#ifndef nsSingleByteCharSetProber_h__ -#define nsSingleByteCharSetProber_h__ - -#include "nsCharSetProber.h" - -#define SAMPLE_SIZE 64 -#define SB_ENOUGH_REL_THRESHOLD 1024 -#define POSITIVE_SHORTCUT_THRESHOLD (float)0.95 -#define NEGATIVE_SHORTCUT_THRESHOLD (float)0.05 -#define SYMBOL_CAT_ORDER 250 -#define NUMBER_OF_SEQ_CAT 4 -#define POSITIVE_CAT (NUMBER_OF_SEQ_CAT-1) -#define NEGATIVE_CAT 0 - -typedef struct -{ - const unsigned char* const charToOrderMap; // [256] table use to find a char's order - const PRUint8* const precedenceMatrix; // [SAMPLE_SIZE][SAMPLE_SIZE]; table to find a 2-char sequence's frequency - float mTypicalPositiveRatio; // = freqSeqs / totalSeqs - PRBool keepEnglishLetter; // says if this script contains English characters (not implemented) - const char* const charsetName; - const char* const langName; -} SequenceModel; - - -class nsSingleByteCharSetProber : public nsCharSetProber{ -public: - nsSingleByteCharSetProber(const SequenceModel *model) - :mModel(model), mReversed(PR_FALSE), mNameProber(0) { Reset(); } - nsSingleByteCharSetProber(const SequenceModel *model, PRBool reversed, nsCharSetProber* nameProber) - :mModel(model), mReversed(reversed), mNameProber(nameProber) { Reset(); } - - virtual const char* GetCharSetName(); - virtual const char* GetLangName() {return mModel?mModel->langName:"";} - virtual nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - virtual nsProbingState GetState(void) {return mState;} - virtual void Reset(void); - virtual float GetConfidence(void); - virtual void SetOpion() {} - - virtual PRBool KeepEnglishLetters() {return mModel->keepEnglishLetter;} - -#ifdef DEBUG_chardet - virtual void DumpStatus(); -#endif - -protected: - nsProbingState mState; - const SequenceModel* const mModel; - const PRBool mReversed; // PR_TRUE if we need to reverse every pair in the model lookup - - //char order of last character - unsigned char mLastOrder; - - PRUint32 mTotalSeqs; - PRUint32 mSeqCounters[NUMBER_OF_SEQ_CAT]; - - PRUint32 mTotalChar; - //characters that fall in our sampling range - PRUint32 mFreqChar; - - // Optional auxiliary prober for name decision. created and destroyed by the GroupProber - nsCharSetProber* mNameProber; - -}; - - -extern const SequenceModel Koi8rModel; -extern const SequenceModel Win1251Model; -extern const SequenceModel Latin5Model; -extern const SequenceModel MacCyrillicModel; -extern const SequenceModel Ibm866Model; -extern const SequenceModel Ibm855Model; -extern const SequenceModel ISO_8859_7greekModel; -extern const SequenceModel WINDOWS_1253greekModel; -extern const SequenceModel Latin5BulgarianModel; -extern const SequenceModel Win1251BulgarianModel; -extern const SequenceModel Latin2HungarianModel; -extern const SequenceModel Win1250HungarianModel; -extern const SequenceModel Win1255Model; -extern const SequenceModel TIS620ThaiModel; -extern const SequenceModel WINDOWS_1252frenchModel; -extern const SequenceModel WINDOWS_1252germanModel; -extern const SequenceModel WINDOWS_1252swedishModel; -extern const SequenceModel ISO_8859_9turkishModel; -extern const SequenceModel WINDOWS_1252finnishModel; -extern const SequenceModel windows_1252spanishModel; -extern const SequenceModel iso_8859_2czechModel; -extern const SequenceModel iso_8859_2polishModel; - -#endif /* nsSingleByteCharSetProber_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSJISProber.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSJISProber.cpp deleted file mode 100644 index 34a9f97..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSJISProber.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -// for S-JIS encoding, obeserve characteristic: -// 1, kana character (or hankaku?) often have hight frequency of appereance -// 2, kana character often exist in group -// 3, certain combination of kana is never used in japanese language - -#include "nsSJISProber.h" -#include "nsDebug.h" - -void nsSJISProber::Reset(void) -{ - mCodingSM->Reset(); - mState = eDetecting; - mContextAnalyser.Reset(mIsPreferredLanguage); - mDistributionAnalyser.Reset(mIsPreferredLanguage); -} - -nsProbingState nsSJISProber::HandleData(const char* aBuf, PRUint32 aLen) -{ - NS_ASSERTION(aLen, "HandleData called with empty buffer"); - nsSMState codingState; - - for (PRUint32 i = 0; i < aLen; i++) - { - codingState = mCodingSM->NextState(aBuf[i]); - if (codingState == eItsMe) - { - mState = eFoundIt; - break; - } - if (codingState == eStart) - { - PRUint32 charLen = mCodingSM->GetCurrentCharLen(); - if (i == 0) - { - mLastChar[1] = aBuf[0]; - mContextAnalyser.HandleOneChar(mLastChar+2-charLen, charLen); - mDistributionAnalyser.HandleOneChar(mLastChar, charLen); - } - else - { - mContextAnalyser.HandleOneChar(aBuf+i+1-charLen, charLen); - mDistributionAnalyser.HandleOneChar(aBuf+i-1, charLen); - } - } - } - - mLastChar[0] = aBuf[aLen-1]; - - if (mState == eDetecting) - if (mContextAnalyser.GotEnoughData() && GetConfidence() > SHORTCUT_THRESHOLD) - mState = eFoundIt; - - return mState; -} - -float nsSJISProber::GetConfidence(void) -{ - float contxtCf = mContextAnalyser.GetConfidence(); - float distribCf = mDistributionAnalyser.GetConfidence(); - - return (contxtCf > distribCf ? contxtCf : distribCf); -} - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSJISProber.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSJISProber.h deleted file mode 100644 index f326ded..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsSJISProber.h +++ /dev/null @@ -1,80 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -// for S-JIS encoding, obeserve characteristic: -// 1, kana character (or hankaku?) often have hight frequency of appereance -// 2, kana character often exist in group -// 3, certain combination of kana is never used in japanese language - -#ifndef nsSJISProber_h__ -#define nsSJISProber_h__ - -#include "nsCharSetProber.h" -#include "nsCodingStateMachine.h" -#include "JpCntx.h" -#include "CharDistribution.h" - - -class nsSJISProber: public nsCharSetProber { -public: - nsSJISProber(PRBool aIsPreferredLanguage) - :mIsPreferredLanguage(aIsPreferredLanguage) - {mCodingSM = new nsCodingStateMachine(&SJISSMModel); - Reset();} - virtual ~nsSJISProber(void){delete mCodingSM;} - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName() {return "SHIFT_JIS";} - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void); - void SetOpion() {} - -protected: - nsCodingStateMachine* mCodingSM; - nsProbingState mState; - - SJISContextAnalysis mContextAnalyser; - SJISDistributionAnalysis mDistributionAnalyser; - - char mLastChar[2]; - PRBool mIsPreferredLanguage; - -}; - - -#endif /* nsSJISProber_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUTF8Prober.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUTF8Prober.cpp deleted file mode 100644 index 7f134a9..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUTF8Prober.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nsUTF8Prober.h" - -void nsUTF8Prober::Reset(void) -{ - mCodingSM->Reset(); - mNumOfMBChar = 0; - mState = eDetecting; -} - -nsProbingState nsUTF8Prober::HandleData(const char* aBuf, PRUint32 aLen) -{ - nsSMState codingState; - - if (mState == eNotMe) - return eNotMe; - - for (PRUint32 i = 0; i < aLen; i++) - { - codingState = mCodingSM->NextState(aBuf[i]); - if (codingState == eItsMe) - { - mState = eFoundIt; - break; - } - if (codingState == eStart) - { - if (mCodingSM->GetCurrentCharLen() >= 2) - mNumOfMBChar++; - } - else if (codingState == eError) - { - return mState = eNotMe; - } - } - - if (mState == eDetecting) - if (GetConfidence() > SHORTCUT_THRESHOLD) - mState = eFoundIt; - return mState; -} - -#define ONE_CHAR_PROB (float)0.50 - -float nsUTF8Prober::GetConfidence(void) -{ - if (mState == eNotMe) - return 0.001; - float unlike = (float)0.99; - - if (mNumOfMBChar < 6) - { - for (PRUint32 i = 0; i < mNumOfMBChar; i++) - unlike *= ONE_CHAR_PROB; - return (float)1.0 - unlike; - } - else - return (float)0.99; -} - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUTF8Prober.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUTF8Prober.h deleted file mode 100644 index 21c91c4..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUTF8Prober.h +++ /dev/null @@ -1,64 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsUTF8Prober_h__ -#define nsUTF8Prober_h__ - -#include "nsCharSetProber.h" -#include "nsCodingStateMachine.h" - -class nsUTF8Prober: public nsCharSetProber { -public: - nsUTF8Prober(){mNumOfMBChar = 0; - mCodingSM = new nsCodingStateMachine(&UTF8SMModel); - Reset(); } - virtual ~nsUTF8Prober(){delete mCodingSM;} - nsProbingState HandleData(const char* aBuf, PRUint32 aLen); - const char* GetCharSetName() {return "UTF-8";} - nsProbingState GetState(void) {return mState;} - void Reset(void); - float GetConfidence(void); - void SetOpion() {} - -protected: - nsCodingStateMachine* mCodingSM; - nsProbingState mState; - PRUint32 mNumOfMBChar; -}; - -#endif /* nsUTF8Prober_h__ */ - diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUniversalDetector.cpp b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUniversalDetector.cpp deleted file mode 100644 index 3e34ded..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUniversalDetector.cpp +++ /dev/null @@ -1,339 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Universal charset detector code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 2001 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shy Shalom - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nscore.h" - -#include "nsUniversalDetector.h" - -#include "nsMBCSGroupProber.h" -#include "nsSBCSGroupProber.h" -#include "nsEscCharsetProber.h" -#include "nsLatin1Prober.h" - -#include - -nsUniversalDetector::nsUniversalDetector(PRUint32 aLanguageFilter) -{ - mDone = PR_FALSE; - mBestGuess = -1; //illegal value as signal - mInTag = PR_FALSE; - mEscCharSetProber = nsnull; - - mStart = PR_TRUE; - mDetectedCharset = nsnull; - mGotData = PR_FALSE; - mInputState = ePureAscii; - mLastChar = '\0'; - mLanguageFilter = aLanguageFilter; - - PRUint32 i; - for (i = 0; i < NUM_OF_CHARSET_PROBERS; i++) - mCharSetProbers[i] = nsnull; -} - -nsUniversalDetector::~nsUniversalDetector() -{ - for (PRInt32 i = 0; i < NUM_OF_CHARSET_PROBERS; i++) - delete mCharSetProbers[i]; - - delete mEscCharSetProber; -} - -void -nsUniversalDetector::Reset() -{ - mDone = PR_FALSE; - mBestGuess = -1; //illegal value as signal - mInTag = PR_FALSE; - - mStart = PR_TRUE; - mDetectedCharset = nsnull; - mGotData = PR_FALSE; - mInputState = ePureAscii; - mLastChar = '\0'; - - if (mEscCharSetProber) - mEscCharSetProber->Reset(); - - PRUint32 i; - for (i = 0; i < NUM_OF_CHARSET_PROBERS; i++) - if (mCharSetProbers[i]) - mCharSetProbers[i]->Reset(); -} - -//--------------------------------------------------------------------- -#define SHORTCUT_THRESHOLD (float)0.95 -#define MINIMUM_THRESHOLD (float)0.20 - -nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen) -{ - if(mDone) - return NS_OK; - - if (aLen > 0) - mGotData = PR_TRUE; - - //If the data starts with BOM, we know it is UTF - if (mStart) - { - mStart = false; - if (aLen >= 2) { - switch (aBuf[0]) { - case '\xEF': - if ((aLen > 2) && ('\xBB' == aBuf[1]) && ('\xBF' == aBuf[2])) { - // EF BB BF UTF-8 encoded BOM - mDetectedCharset = "UTF-8"; - } - break; - case '\xFE': - if ('\xFF' == aBuf[1]) { - // FE FF UTF-16, big endian BOM - mDetectedCharset = "UTF-16BE"; - } - break; - case '\xFF': - if ('\xFE' == aBuf[1]) { - // FF FE UTF-16, little endian BOM - mDetectedCharset = "UTF-16LE"; - } - break; - } // switch - } - - if (mDetectedCharset) - { - mDone = PR_TRUE; - return NS_OK; - } - } - - PRUint32 i; - for (i = 0; i < aLen; i++) - { - //other than 0xa0, if every othe character is ascii, the page is ascii - if ((aBuf[i] & '\x80') && aBuf[i] != '\xA0') //Since many Ascii only page contains NBSP - { - //we got a non-ascii byte (high-byte) - if (mInputState != eHighbyte) - { - //adjust state - mInputState = eHighbyte; - - //kill mEscCharSetProber if it is active - if (mEscCharSetProber) { - delete mEscCharSetProber; - mEscCharSetProber = nsnull; - } - - //start multibyte and singlebyte charset prober - if (nsnull == mCharSetProbers[0]) - { - mCharSetProbers[0] = new nsMBCSGroupProber(mLanguageFilter); - if (nsnull == mCharSetProbers[0]) - return NS_ERROR_OUT_OF_MEMORY; - } - if (nsnull == mCharSetProbers[1] && - (mLanguageFilter & NS_FILTER_NON_CJK)) - { - mCharSetProbers[1] = new nsSBCSGroupProber; - if (nsnull == mCharSetProbers[1]) - return NS_ERROR_OUT_OF_MEMORY; - } - if (nsnull == mCharSetProbers[2]) - { - mCharSetProbers[2] = new nsLatin1Prober; - if (nsnull == mCharSetProbers[2]) - return NS_ERROR_OUT_OF_MEMORY; - } - } - } - else - { - //ok, just pure ascii so far - if ( ePureAscii == mInputState && - (aBuf[i] == '\033' || (aBuf[i] == '{' && mLastChar == '~')) ) - { - //found escape character or HZ "~{" - mInputState = eEscAscii; - } - mLastChar = aBuf[i]; - } - } - - nsProbingState st; - switch (mInputState) - { - case eEscAscii: - if (nsnull == mEscCharSetProber) { - mEscCharSetProber = new nsEscCharSetProber(mLanguageFilter); - if (nsnull == mEscCharSetProber) - return NS_ERROR_OUT_OF_MEMORY; - } - st = mEscCharSetProber->HandleData(aBuf, aLen); - if (st == eFoundIt) - { - mDone = PR_TRUE; - mDetectedCharset = mEscCharSetProber->GetCharSetName(); - } - break; - case eHighbyte: - for (i = 0; i < NUM_OF_CHARSET_PROBERS; i++) - { - if (mCharSetProbers[i]) - { - st = mCharSetProbers[i]->HandleData(aBuf, aLen); - if (st == eFoundIt) - { - mDone = PR_TRUE; - mDetectedCharset = mCharSetProbers[i]->GetCharSetName(); - return NS_OK; - } - } - } - break; - - default: //pure ascii - ;//do nothing here - } - return NS_OK; -} - -//--------------------------------------------------------------------- -void nsUniversalDetector::DataEnd() -{ - if (!mGotData) - { - // we haven't got any data yet, return immediately - // caller program sometimes call DataEnd before anything has been sent to detector - return; - } - - if (mDetectedCharset) - { - mDone = PR_TRUE; - Report(mDetectedCharset); - return; - } - - switch (mInputState) - { - case eHighbyte: - { - float proberConfidence; - float maxProberConfidence = (float)0.0; - PRInt32 maxProber = 0; - - for (PRInt32 i = 0; i < NUM_OF_CHARSET_PROBERS; i++) - { - if (mCharSetProbers[i]) - { - proberConfidence = mCharSetProbers[i]->GetConfidence(); - if (proberConfidence > maxProberConfidence) - { - maxProberConfidence = proberConfidence; - maxProber = i; - } - } - } - //do not report anything because we are not confident of it, that's in fact a negative answer - if (maxProberConfidence > MINIMUM_THRESHOLD) - Report(mCharSetProbers[maxProber]->GetCharSetName()); - } - break; - case eEscAscii: - break; - default: - ; - } - return; -} - -///* -void nsUniversalDetector::DataEnd2(float *confidence) -{ - if (!mGotData) - { - // we haven't got any data yet, return immediately - // caller program sometimes call DataEnd before anything has been sent to detector - return; - } - - if (mDetectedCharset) - { - mDone = PR_TRUE; - Report(mDetectedCharset); - // no confidence - *confidence = (float)0.99; - return; - } - - switch (mInputState) - { - case eHighbyte: - { - float proberConfidence; - float maxProberConfidence = (float)0.0; - PRInt32 maxProber = 0; - - for (PRInt32 i = 0; i < NUM_OF_CHARSET_PROBERS; i++) - { - if (mCharSetProbers[i]) - { - proberConfidence = mCharSetProbers[i]->GetConfidence(); - if (proberConfidence > maxProberConfidence) - { - maxProberConfidence = proberConfidence; - maxProber = i; - } - } - } - //do not report anything because we are not confident of it, that's in fact a negative answer - if (maxProberConfidence > MINIMUM_THRESHOLD){ - Report(mCharSetProbers[maxProber]->GetCharSetName()); - *confidence = maxProberConfidence; - } - } - break; - case eEscAscii: - break; - default: - ; - } - return; -} -//*/ \ No newline at end of file diff --git a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUniversalDetector.h b/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUniversalDetector.h deleted file mode 100644 index 598b813..0000000 --- a/src/ext/libcharsetdetect/mozilla/extensions/universalchardet/src/base/nsUniversalDetector.h +++ /dev/null @@ -1,91 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Mozilla Communicator client code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsUniversalDetector_h__ -#define nsUniversalDetector_h__ - -class nsCharSetProber; - -#define NUM_OF_CHARSET_PROBERS 3 - -typedef enum { - ePureAscii = 0, - eEscAscii = 1, - eHighbyte = 2 -} nsInputState; - -#define NS_FILTER_CHINESE_SIMPLIFIED 0x01 -#define NS_FILTER_CHINESE_TRADITIONAL 0x02 -#define NS_FILTER_JAPANESE 0x04 -#define NS_FILTER_KOREAN 0x08 -#define NS_FILTER_NON_CJK 0x10 -#define NS_FILTER_ALL 0x1F -#define NS_FILTER_CHINESE (NS_FILTER_CHINESE_SIMPLIFIED | \ - NS_FILTER_CHINESE_TRADITIONAL) -#define NS_FILTER_CJK (NS_FILTER_CHINESE_SIMPLIFIED | \ - NS_FILTER_CHINESE_TRADITIONAL | \ - NS_FILTER_JAPANESE | \ - NS_FILTER_KOREAN) - -class nsUniversalDetector { -public: - nsUniversalDetector(PRUint32 aLanguageFilter); - virtual ~nsUniversalDetector(); - virtual nsresult HandleData(const char* aBuf, PRUint32 aLen); - virtual void DataEnd(void); - - virtual void DataEnd2(float *confidence); - -protected: - virtual void Report(const char* aCharset) = 0; - virtual void Reset(); - nsInputState mInputState; - PRBool mDone; - PRBool mInTag; - PRBool mStart; - PRBool mGotData; - char mLastChar; - const char * mDetectedCharset; - PRInt32 mBestGuess; - PRUint32 mLanguageFilter; - - nsCharSetProber *mCharSetProbers[NUM_OF_CHARSET_PROBERS]; - nsCharSetProber *mEscCharSetProber; -}; - -#endif - diff --git a/src/ext/libcharsetdetect/nscore.h b/src/ext/libcharsetdetect/nscore.h deleted file mode 100644 index 6faac93..0000000 --- a/src/ext/libcharsetdetect/nscore.h +++ /dev/null @@ -1,6 +0,0 @@ -#include "prtypes.h" -#define NS_OK 0 -#define nsnull 0L -typedef PRUint32 nsresult; -#define NS_ERROR_OUT_OF_MEMORY ((nsresult) 0x8007000eL) -#define NS_ASSERTION(A,B) diff --git a/src/ext/libcharsetdetect/nspr-emu/README.md b/src/ext/libcharsetdetect/nspr-emu/README.md deleted file mode 100644 index acead14..0000000 --- a/src/ext/libcharsetdetect/nspr-emu/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# NSPR emulation library - -Author: Max Bolingbroke () - -This library was created by pulling together the minimal pieces of NSPR required to get the Universal Character Set -Detector (UCSD) to compile. It turns out that the UCSD uses only very minimal parts of NSPR, so we can get away with just a few -header files and totally forget about linking to the library. - -Precisely, we used these files from [NSPR v4.8.6](https://ftp.mozilla.org/pub/mozilla.org/nspr/releases/v4.8.6/): - - * prtypes.h - * prmem.h - * obsolete/protypes.h - -I then patched some prototypes in prmem.h to use C library functions directly. The protypes were replaced with #define directives -that ensure that callers of PR_Malloc, PR_Calloc, PR_Realloc, PR_Free just use their C library counterparts directly. - -To finish off, I needed to provide prcpucfg.h. Normally this is generated by the nspr build process, but to simplify things I used -the version from [Google's Chrome source tree](http://src.chromium.org/viewvc/chrome/trunk/src/base/third_party/nspr/prcpucfg.h?view=markup) -that just uses the preprocessor to choose between several pre-generated versions. - -I had to patch the Google-provided prcpucfg.h so that the #included paths pointed to the right directory. - -## Licensing - -Chromium (and thus presumably their contribution) is licensed under the BSD license. - -NSPR is licensed under the MPL/GPL/LGPL tri-license. - -Thus, this emulation library is licensed under the LGPL (I think). \ No newline at end of file diff --git a/src/ext/libcharsetdetect/nspr-emu/nsDebug.h b/src/ext/libcharsetdetect/nspr-emu/nsDebug.h deleted file mode 100644 index e69de29..0000000 diff --git a/src/ext/libcharsetdetect/nspr-emu/obsolete/protypes.h b/src/ext/libcharsetdetect/nspr-emu/obsolete/protypes.h deleted file mode 100644 index d8e947a..0000000 --- a/src/ext/libcharsetdetect/nspr-emu/obsolete/protypes.h +++ /dev/null @@ -1,231 +0,0 @@ -/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is the Netscape Portable Runtime (NSPR). - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998-2000 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -/* - * This header typedefs the old 'native' types to the new PRs. - * These definitions are scheduled to be eliminated at the earliest - * possible time. The NSPR API is implemented and documented using - * the new definitions. - */ - -#if !defined(PROTYPES_H) -#define PROTYPES_H - -typedef PRUintn uintn; -#ifndef _XP_Core_ -typedef PRIntn intn; -#endif - -/* - * It is trickier to define uint, int8, uint8, int16, uint16, - * int32, uint32, int64, and uint64 because some of these int - * types are defined by standard header files on some platforms. - * Our strategy here is to include all such standard headers - * first, and then define these int types only if they are not - * defined by those standard headers. - */ - -/* - * BeOS defines all the int types below in its standard header - * file SupportDefs.h. - */ -#ifdef XP_BEOS -#include -#endif - -/* - * SVR4 typedef of uint is commonly found on UNIX machines. - * - * On AIX 4.3, sys/inttypes.h (which is included by sys/types.h) - * defines the types int8, int16, int32, and int64. - * - * On OS/2, sys/types.h defines uint. - */ -#if defined(XP_UNIX) || defined(XP_OS2) -#include -#endif - -/* model.h on HP-UX defines int8, int16, and int32. */ -#ifdef HPUX -#include -#endif - -/* - * uint - */ - -#if !defined(XP_BEOS) && !defined(XP_OS2) && !defined(XP_UNIX) || defined(NTO) -typedef PRUintn uint; -#endif - -/* - * uint64 - */ - -#if !defined(XP_BEOS) -typedef PRUint64 uint64; -#endif - -/* - * uint32 - */ - -#if !defined(XP_BEOS) -#if !defined(_WIN32) && !defined(XP_OS2) && !defined(NTO) -typedef PRUint32 uint32; -#else -typedef unsigned long uint32; -#endif -#endif - -/* - * uint16 - */ - -#if !defined(XP_BEOS) -typedef PRUint16 uint16; -#endif - -/* - * uint8 - */ - -#if !defined(XP_BEOS) -typedef PRUint8 uint8; -#endif - -/* - * int64 - */ - -#if !defined(XP_BEOS) && !defined(_PR_AIX_HAVE_BSD_INT_TYPES) -typedef PRInt64 int64; -#endif - -/* - * int32 - */ - -#if !defined(XP_BEOS) && !defined(_PR_AIX_HAVE_BSD_INT_TYPES) \ - && !defined(HPUX) -#if !defined(_WIN32) && !defined(XP_OS2) && !defined(NTO) -typedef PRInt32 int32; -#else -typedef long int32; -#endif -#endif - -/* - * int16 - */ - -#if !defined(XP_BEOS) && !defined(_PR_AIX_HAVE_BSD_INT_TYPES) \ - && !defined(HPUX) -typedef PRInt16 int16; -#endif - -/* - * int8 - */ - -#if !defined(XP_BEOS) && !defined(_PR_AIX_HAVE_BSD_INT_TYPES) \ - && !defined(HPUX) -typedef PRInt8 int8; -#endif - -typedef PRFloat64 float64; -typedef PRUptrdiff uptrdiff_t; -typedef PRUword uprword_t; -typedef PRWord prword_t; - - -/* Re: prbit.h */ -#define TEST_BIT PR_TEST_BIT -#define SET_BIT PR_SET_BIT -#define CLEAR_BIT PR_CLEAR_BIT - -/* Re: prarena.h->plarena.h */ -#define PRArena PLArena -#define PRArenaPool PLArenaPool -#define PRArenaStats PLArenaStats -#define PR_ARENA_ALIGN PL_ARENA_ALIGN -#define PR_INIT_ARENA_POOL PL_INIT_ARENA_POOL -#define PR_ARENA_ALLOCATE PL_ARENA_ALLOCATE -#define PR_ARENA_GROW PL_ARENA_GROW -#define PR_ARENA_MARK PL_ARENA_MARK -#define PR_CLEAR_UNUSED PL_CLEAR_UNUSED -#define PR_CLEAR_ARENA PL_CLEAR_ARENA -#define PR_ARENA_RELEASE PL_ARENA_RELEASE -#define PR_COUNT_ARENA PL_COUNT_ARENA -#define PR_ARENA_DESTROY PL_ARENA_DESTROY -#define PR_InitArenaPool PL_InitArenaPool -#define PR_FreeArenaPool PL_FreeArenaPool -#define PR_FinishArenaPool PL_FinishArenaPool -#define PR_CompactArenaPool PL_CompactArenaPool -#define PR_ArenaFinish PL_ArenaFinish -#define PR_ArenaAllocate PL_ArenaAllocate -#define PR_ArenaGrow PL_ArenaGrow -#define PR_ArenaRelease PL_ArenaRelease -#define PR_ArenaCountAllocation PL_ArenaCountAllocation -#define PR_ArenaCountInplaceGrowth PL_ArenaCountInplaceGrowth -#define PR_ArenaCountGrowth PL_ArenaCountGrowth -#define PR_ArenaCountRelease PL_ArenaCountRelease -#define PR_ArenaCountRetract PL_ArenaCountRetract - -/* Re: prhash.h->plhash.h */ -#define PRHashEntry PLHashEntry -#define PRHashTable PLHashTable -#define PRHashNumber PLHashNumber -#define PRHashFunction PLHashFunction -#define PRHashComparator PLHashComparator -#define PRHashEnumerator PLHashEnumerator -#define PRHashAllocOps PLHashAllocOps -#define PR_NewHashTable PL_NewHashTable -#define PR_HashTableDestroy PL_HashTableDestroy -#define PR_HashTableRawLookup PL_HashTableRawLookup -#define PR_HashTableRawAdd PL_HashTableRawAdd -#define PR_HashTableRawRemove PL_HashTableRawRemove -#define PR_HashTableAdd PL_HashTableAdd -#define PR_HashTableRemove PL_HashTableRemove -#define PR_HashTableEnumerateEntries PL_HashTableEnumerateEntries -#define PR_HashTableLookup PL_HashTableLookup -#define PR_HashTableDump PL_HashTableDump -#define PR_HashString PL_HashString -#define PR_CompareStrings PL_CompareStrings -#define PR_CompareValues PL_CompareValues - -#endif /* !defined(PROTYPES_H) */ diff --git a/src/ext/libcharsetdetect/nspr-emu/prcpucfg.h b/src/ext/libcharsetdetect/nspr-emu/prcpucfg.h deleted file mode 100644 index 4dad9a5..0000000 --- a/src/ext/libcharsetdetect/nspr-emu/prcpucfg.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef BASE_THIRD_PARTY_NSPR_PRCPUCFG_H__ -#define BASE_THIRD_PARTY_NSPR_PRCPUCFG_H__ - -#if defined(WIN32) -#include "prcpucfg_win.h" -#elif defined(__APPLE__) -#include "prcpucfg_mac.h" -#elif defined(__linux__) -#include "prcpucfg_linux.h" -#elif defined(__FreeBSD__) -#include "prcpucfg_freebsd.h" -#elif defined(__OpenBSD__) -#include "prcpucfg_openbsd.h" -#elif defined(__sun) -#include "prcpucfg_solaris.h" -#else -#error Provide a prcpucfg.h appropriate for your platform -#endif - -#endif // BASE_THIRD_PARTY_NSPR_PRCPUCFG_H__ diff --git a/src/ext/libcharsetdetect/nspr-emu/prcpucfg_freebsd.h b/src/ext/libcharsetdetect/nspr-emu/prcpucfg_freebsd.h deleted file mode 100644 index 76d3542..0000000 --- a/src/ext/libcharsetdetect/nspr-emu/prcpucfg_freebsd.h +++ /dev/null @@ -1,337 +0,0 @@ -/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is the Netscape Portable Runtime (NSPR). - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998-2000 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nspr_cpucfg___ -#define nspr_cpucfg___ - -#ifndef XP_UNIX -#define XP_UNIX -#endif - -#ifndef FREEBSD -#define FREEBSD -#endif - -#define PR_AF_INET6 28 /* same as AF_INET6 */ - -#ifndef HAVE_LONG_LONG -#define HAVE_LONG_LONG -#endif - -#if defined(__i386__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#undef HAVE_ALIGNED_DOUBLES -#undef HAVE_ALIGNED_LONGLONGS - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 4 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 4 -#define PR_ALIGN_OF_POINTER 4 - -#elif defined(__alpha__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define HAVE_ALIGNED_DOUBLES -#define HAVE_ALIGNED_LONGLONGS -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 - -#elif defined(__sparc__) - -#undef IS_LITTLE_ENDIAN -#define IS_BIG_ENDIAN 1 -#define HAVE_ALIGNED_DOUBLES -#define HAVE_ALIGNED_LONGLONGS -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 - -#elif defined(__ia64__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define HAVE_ALIGNED_DOUBLES -#define HAVE_ALIGNED_LONGLONGS -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 -#define PR_ALIGN_OF_WORD 8 - -#elif defined(__amd64__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define HAVE_ALIGNED_DOUBLES -#define HAVE_ALIGNED_LONGLONGS -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 -#define PR_ALIGN_OF_WORD 8 - -#else - -#error "Unknown CPU architecture" - -#endif - -#ifndef NO_NSPR_10_SUPPORT - -#define BYTES_PER_BYTE PR_BYTES_PER_BYTE -#define BYTES_PER_SHORT PR_BYTES_PER_SHORT -#define BYTES_PER_INT PR_BYTES_PER_INT -#define BYTES_PER_INT64 PR_BYTES_PER_INT64 -#define BYTES_PER_LONG PR_BYTES_PER_LONG -#define BYTES_PER_FLOAT PR_BYTES_PER_FLOAT -#define BYTES_PER_DOUBLE PR_BYTES_PER_DOUBLE -#define BYTES_PER_WORD PR_BYTES_PER_WORD -#define BYTES_PER_DWORD PR_BYTES_PER_DWORD - -#define BITS_PER_BYTE PR_BITS_PER_BYTE -#define BITS_PER_SHORT PR_BITS_PER_SHORT -#define BITS_PER_INT PR_BITS_PER_INT -#define BITS_PER_INT64 PR_BITS_PER_INT64 -#define BITS_PER_LONG PR_BITS_PER_LONG -#define BITS_PER_FLOAT PR_BITS_PER_FLOAT -#define BITS_PER_DOUBLE PR_BITS_PER_DOUBLE -#define BITS_PER_WORD PR_BITS_PER_WORD - -#define BITS_PER_BYTE_LOG2 PR_BITS_PER_BYTE_LOG2 -#define BITS_PER_SHORT_LOG2 PR_BITS_PER_SHORT_LOG2 -#define BITS_PER_INT_LOG2 PR_BITS_PER_INT_LOG2 -#define BITS_PER_INT64_LOG2 PR_BITS_PER_INT64_LOG2 -#define BITS_PER_LONG_LOG2 PR_BITS_PER_LONG_LOG2 -#define BITS_PER_FLOAT_LOG2 PR_BITS_PER_FLOAT_LOG2 -#define BITS_PER_DOUBLE_LOG2 PR_BITS_PER_DOUBLE_LOG2 -#define BITS_PER_WORD_LOG2 PR_BITS_PER_WORD_LOG2 - -#define ALIGN_OF_SHORT PR_ALIGN_OF_SHORT -#define ALIGN_OF_INT PR_ALIGN_OF_INT -#define ALIGN_OF_LONG PR_ALIGN_OF_LONG -#define ALIGN_OF_INT64 PR_ALIGN_OF_INT64 -#define ALIGN_OF_FLOAT PR_ALIGN_OF_FLOAT -#define ALIGN_OF_DOUBLE PR_ALIGN_OF_DOUBLE -#define ALIGN_OF_POINTER PR_ALIGN_OF_POINTER -#define ALIGN_OF_WORD PR_ALIGN_OF_WORD - -#define BYTES_PER_WORD_LOG2 PR_BYTES_PER_WORD_LOG2 -#define BYTES_PER_DWORD_LOG2 PR_BYTES_PER_DWORD_LOG2 -#define WORDS_PER_DWORD_LOG2 PR_WORDS_PER_DWORD_LOG2 - -#endif /* NO_NSPR_10_SUPPORT */ - -#endif /* nspr_cpucfg___ */ diff --git a/src/ext/libcharsetdetect/nspr-emu/prcpucfg_linux.h b/src/ext/libcharsetdetect/nspr-emu/prcpucfg_linux.h deleted file mode 100644 index 2a6b66f..0000000 --- a/src/ext/libcharsetdetect/nspr-emu/prcpucfg_linux.h +++ /dev/null @@ -1,753 +0,0 @@ -/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is the Netscape Portable Runtime (NSPR). - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998-2000 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nspr_cpucfg___ -#define nspr_cpucfg___ - -#ifndef XP_UNIX -#define XP_UNIX -#endif - -#ifndef LINUX -#define LINUX -#endif - -#define PR_AF_INET6 10 /* same as AF_INET6 */ - -#ifdef __powerpc64__ - -#undef IS_LITTLE_ENDIAN -#define IS_BIG_ENDIAN 1 -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 -#define PR_ALIGN_OF_WORD 8 - -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__powerpc__) - -#undef IS_LITTLE_ENDIAN -#define IS_BIG_ENDIAN 1 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 4 -#define PR_ALIGN_OF_WORD 4 - -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__alpha) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 -#define PR_ALIGN_OF_WORD 8 - -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__ia64__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 -#define PR_ALIGN_OF_WORD 8 - -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__x86_64__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 -#define PR_ALIGN_OF_WORD 8 - -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__mc68000__) - -#undef IS_LITTLE_ENDIAN -#define IS_BIG_ENDIAN 1 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 2 -#define PR_ALIGN_OF_LONG 2 -#define PR_ALIGN_OF_INT64 2 -#define PR_ALIGN_OF_FLOAT 2 -#define PR_ALIGN_OF_DOUBLE 2 -#define PR_ALIGN_OF_POINTER 2 -#define PR_ALIGN_OF_WORD 2 - -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__sparc__) - -#undef IS_LITTLE_ENDIAN -#define IS_BIG_ENDIAN 1 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 4 -#define PR_ALIGN_OF_WORD 4 - -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__i386__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 4 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 4 -#define PR_ALIGN_OF_POINTER 4 -#define PR_ALIGN_OF_WORD 4 - -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__mips__) - -#ifdef __MIPSEB__ -#define IS_BIG_ENDIAN 1 -#undef IS_LITTLE_ENDIAN -#elif defined(__MIPSEL__) -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#else -#error "Unknown MIPS endianness." -#endif - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 4 -#define PR_ALIGN_OF_WORD 4 - -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__arm__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 4 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 4 -#define PR_ALIGN_OF_POINTER 4 -#define PR_ALIGN_OF_WORD 4 - -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__aarch64__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 -#define PR_ALIGN_OF_WORD 8 - -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__hppa__) - -#undef IS_LITTLE_ENDIAN -#define IS_BIG_ENDIAN 1 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 4 -#define PR_ALIGN_OF_WORD 4 - -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__s390x__) - -#define IS_BIG_ENDIAN 1 -#undef IS_LITTLE_ENDIAN -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 -#define PR_ALIGN_OF_WORD 8 - -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(__s390__) - -#define IS_BIG_ENDIAN 1 -#undef IS_LITTLE_ENDIAN - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 4 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 4 -#define PR_ALIGN_OF_POINTER 4 -#define PR_ALIGN_OF_WORD 4 - -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#else - -#error "Unknown CPU architecture" - -#endif - -#define HAVE_LONG_LONG -#if PR_ALIGN_OF_DOUBLE == 8 -#define HAVE_ALIGNED_DOUBLES -#endif -#if PR_ALIGN_OF_INT64 == 8 -#define HAVE_ALIGNED_LONGLONGS -#endif - -#ifndef NO_NSPR_10_SUPPORT - -#define BYTES_PER_BYTE PR_BYTES_PER_BYTE -#define BYTES_PER_SHORT PR_BYTES_PER_SHORT -#define BYTES_PER_INT PR_BYTES_PER_INT -#define BYTES_PER_INT64 PR_BYTES_PER_INT64 -#define BYTES_PER_LONG PR_BYTES_PER_LONG -#define BYTES_PER_FLOAT PR_BYTES_PER_FLOAT -#define BYTES_PER_DOUBLE PR_BYTES_PER_DOUBLE -#define BYTES_PER_WORD PR_BYTES_PER_WORD -#define BYTES_PER_DWORD PR_BYTES_PER_DWORD - -#define BITS_PER_BYTE PR_BITS_PER_BYTE -#define BITS_PER_SHORT PR_BITS_PER_SHORT -#define BITS_PER_INT PR_BITS_PER_INT -#define BITS_PER_INT64 PR_BITS_PER_INT64 -#define BITS_PER_LONG PR_BITS_PER_LONG -#define BITS_PER_FLOAT PR_BITS_PER_FLOAT -#define BITS_PER_DOUBLE PR_BITS_PER_DOUBLE -#define BITS_PER_WORD PR_BITS_PER_WORD - -#define BITS_PER_BYTE_LOG2 PR_BITS_PER_BYTE_LOG2 -#define BITS_PER_SHORT_LOG2 PR_BITS_PER_SHORT_LOG2 -#define BITS_PER_INT_LOG2 PR_BITS_PER_INT_LOG2 -#define BITS_PER_INT64_LOG2 PR_BITS_PER_INT64_LOG2 -#define BITS_PER_LONG_LOG2 PR_BITS_PER_LONG_LOG2 -#define BITS_PER_FLOAT_LOG2 PR_BITS_PER_FLOAT_LOG2 -#define BITS_PER_DOUBLE_LOG2 PR_BITS_PER_DOUBLE_LOG2 -#define BITS_PER_WORD_LOG2 PR_BITS_PER_WORD_LOG2 - -#define ALIGN_OF_SHORT PR_ALIGN_OF_SHORT -#define ALIGN_OF_INT PR_ALIGN_OF_INT -#define ALIGN_OF_LONG PR_ALIGN_OF_LONG -#define ALIGN_OF_INT64 PR_ALIGN_OF_INT64 -#define ALIGN_OF_FLOAT PR_ALIGN_OF_FLOAT -#define ALIGN_OF_DOUBLE PR_ALIGN_OF_DOUBLE -#define ALIGN_OF_POINTER PR_ALIGN_OF_POINTER -#define ALIGN_OF_WORD PR_ALIGN_OF_WORD - -#define BYTES_PER_WORD_LOG2 PR_BYTES_PER_WORD_LOG2 -#define BYTES_PER_DWORD_LOG2 PR_BYTES_PER_DWORD_LOG2 -#define WORDS_PER_DWORD_LOG2 PR_WORDS_PER_DWORD_LOG2 - -#endif /* NO_NSPR_10_SUPPORT */ - -#endif /* nspr_cpucfg___ */ diff --git a/src/ext/libcharsetdetect/nspr-emu/prcpucfg_mac.h b/src/ext/libcharsetdetect/nspr-emu/prcpucfg_mac.h deleted file mode 100644 index dc7e0e0..0000000 --- a/src/ext/libcharsetdetect/nspr-emu/prcpucfg_mac.h +++ /dev/null @@ -1,145 +0,0 @@ -/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is the Netscape Portable Runtime (NSPR). - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998-2000 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nspr_cpucfg___ -#define nspr_cpucfg___ - -#ifndef XP_UNIX -#define XP_UNIX -#endif - -#define PR_AF_INET6 30 /* same as AF_INET6 */ - -#if defined(i386) -#undef IS_BIG_ENDIAN -#define IS_LITTLE_ENDIAN 1 -#else -#undef IS_LITTLE_ENDIAN -#define IS_BIG_ENDIAN 1 -#endif - -#define HAVE_LONG_LONG -#undef HAVE_ALIGNED_DOUBLES -#define HAVE_ALIGNED_LONGLONGS 1 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 -#define PR_BITS_PER_DWORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 4 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 4 -#define PR_ALIGN_OF_POINTER 4 -#define PR_ALIGN_OF_WORD 4 - -#ifndef NO_NSPR_10_SUPPORT - -#define BYTES_PER_BYTE PR_BYTES_PER_BYTE -#define BYTES_PER_SHORT PR_BYTES_PER_SHORT -#define BYTES_PER_INT PR_BYTES_PER_INT -#define BYTES_PER_INT64 PR_BYTES_PER_INT64 -#define BYTES_PER_LONG PR_BYTES_PER_LONG -#define BYTES_PER_FLOAT PR_BYTES_PER_FLOAT -#define BYTES_PER_DOUBLE PR_BYTES_PER_DOUBLE -#define BYTES_PER_WORD PR_BYTES_PER_WORD -#define BYTES_PER_DWORD PR_BYTES_PER_DWORD - -#define BITS_PER_BYTE PR_BITS_PER_BYTE -#define BITS_PER_SHORT PR_BITS_PER_SHORT -#define BITS_PER_INT PR_BITS_PER_INT -#define BITS_PER_INT64 PR_BITS_PER_INT64 -#define BITS_PER_LONG PR_BITS_PER_LONG -#define BITS_PER_FLOAT PR_BITS_PER_FLOAT -#define BITS_PER_DOUBLE PR_BITS_PER_DOUBLE -#define BITS_PER_WORD PR_BITS_PER_WORD - -#define BITS_PER_BYTE_LOG2 PR_BITS_PER_BYTE_LOG2 -#define BITS_PER_SHORT_LOG2 PR_BITS_PER_SHORT_LOG2 -#define BITS_PER_INT_LOG2 PR_BITS_PER_INT_LOG2 -#define BITS_PER_INT64_LOG2 PR_BITS_PER_INT64_LOG2 -#define BITS_PER_LONG_LOG2 PR_BITS_PER_LONG_LOG2 -#define BITS_PER_FLOAT_LOG2 PR_BITS_PER_FLOAT_LOG2 -#define BITS_PER_DOUBLE_LOG2 PR_BITS_PER_DOUBLE_LOG2 -#define BITS_PER_WORD_LOG2 PR_BITS_PER_WORD_LOG2 - -#define ALIGN_OF_SHORT PR_ALIGN_OF_SHORT -#define ALIGN_OF_INT PR_ALIGN_OF_INT -#define ALIGN_OF_LONG PR_ALIGN_OF_LONG -#define ALIGN_OF_INT64 PR_ALIGN_OF_INT64 -#define ALIGN_OF_FLOAT PR_ALIGN_OF_FLOAT -#define ALIGN_OF_DOUBLE PR_ALIGN_OF_DOUBLE -#define ALIGN_OF_POINTER PR_ALIGN_OF_POINTER -#define ALIGN_OF_WORD PR_ALIGN_OF_WORD - -#define BYTES_PER_WORD_LOG2 PR_BYTES_PER_WORD_LOG2 -#define BYTES_PER_DWORD_LOG2 PR_BYTES_PER_DWORD_LOG2 -#define WORDS_PER_DWORD_LOG2 PR_WORDS_PER_DWORD_LOG2 - -#endif /* NO_NSPR_10_SUPPORT */ - -#endif /* nspr_cpucfg___ */ - diff --git a/src/ext/libcharsetdetect/nspr-emu/prcpucfg_openbsd.h b/src/ext/libcharsetdetect/nspr-emu/prcpucfg_openbsd.h deleted file mode 100644 index 93c5b32..0000000 --- a/src/ext/libcharsetdetect/nspr-emu/prcpucfg_openbsd.h +++ /dev/null @@ -1,337 +0,0 @@ -/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is the Netscape Portable Runtime (NSPR). - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998-2000 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nspr_cpucfg___ -#define nspr_cpucfg___ - -#ifndef XP_UNIX -#define XP_UNIX -#endif - -#ifndef OPENBSD -#define OPENBSD -#endif - -#define PR_AF_INET6 28 /* same as AF_INET6 */ - -#ifndef HAVE_LONG_LONG -#define HAVE_LONG_LONG -#endif - -#if defined(__i386__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#undef HAVE_ALIGNED_DOUBLES -#undef HAVE_ALIGNED_LONGLONGS - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 32 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 5 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 4 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 4 -#define PR_ALIGN_OF_POINTER 4 - -#elif defined(__alpha__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define HAVE_ALIGNED_DOUBLES -#define HAVE_ALIGNED_LONGLONGS -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 - -#elif defined(__sparc__) - -#undef IS_LITTLE_ENDIAN -#define IS_BIG_ENDIAN 1 -#define HAVE_ALIGNED_DOUBLES -#define HAVE_ALIGNED_LONGLONGS -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 - -#elif defined(__ia64__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define HAVE_ALIGNED_DOUBLES -#define HAVE_ALIGNED_LONGLONGS -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 -#define PR_ALIGN_OF_WORD 8 - -#elif defined(__amd64__) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define HAVE_ALIGNED_DOUBLES -#define HAVE_ALIGNED_LONGLONGS -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 8 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_DOUBLE 8 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 64 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_DOUBLE 64 -#define PR_BITS_PER_WORD 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 6 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_DOUBLE_LOG2 6 -#define PR_BITS_PER_WORD_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 8 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 -#define PR_ALIGN_OF_WORD 8 - -#else - -#error "Unknown CPU architecture" - -#endif - -#ifndef NO_NSPR_10_SUPPORT - -#define BYTES_PER_BYTE PR_BYTES_PER_BYTE -#define BYTES_PER_SHORT PR_BYTES_PER_SHORT -#define BYTES_PER_INT PR_BYTES_PER_INT -#define BYTES_PER_INT64 PR_BYTES_PER_INT64 -#define BYTES_PER_LONG PR_BYTES_PER_LONG -#define BYTES_PER_FLOAT PR_BYTES_PER_FLOAT -#define BYTES_PER_DOUBLE PR_BYTES_PER_DOUBLE -#define BYTES_PER_WORD PR_BYTES_PER_WORD -#define BYTES_PER_DWORD PR_BYTES_PER_DWORD - -#define BITS_PER_BYTE PR_BITS_PER_BYTE -#define BITS_PER_SHORT PR_BITS_PER_SHORT -#define BITS_PER_INT PR_BITS_PER_INT -#define BITS_PER_INT64 PR_BITS_PER_INT64 -#define BITS_PER_LONG PR_BITS_PER_LONG -#define BITS_PER_FLOAT PR_BITS_PER_FLOAT -#define BITS_PER_DOUBLE PR_BITS_PER_DOUBLE -#define BITS_PER_WORD PR_BITS_PER_WORD - -#define BITS_PER_BYTE_LOG2 PR_BITS_PER_BYTE_LOG2 -#define BITS_PER_SHORT_LOG2 PR_BITS_PER_SHORT_LOG2 -#define BITS_PER_INT_LOG2 PR_BITS_PER_INT_LOG2 -#define BITS_PER_INT64_LOG2 PR_BITS_PER_INT64_LOG2 -#define BITS_PER_LONG_LOG2 PR_BITS_PER_LONG_LOG2 -#define BITS_PER_FLOAT_LOG2 PR_BITS_PER_FLOAT_LOG2 -#define BITS_PER_DOUBLE_LOG2 PR_BITS_PER_DOUBLE_LOG2 -#define BITS_PER_WORD_LOG2 PR_BITS_PER_WORD_LOG2 - -#define ALIGN_OF_SHORT PR_ALIGN_OF_SHORT -#define ALIGN_OF_INT PR_ALIGN_OF_INT -#define ALIGN_OF_LONG PR_ALIGN_OF_LONG -#define ALIGN_OF_INT64 PR_ALIGN_OF_INT64 -#define ALIGN_OF_FLOAT PR_ALIGN_OF_FLOAT -#define ALIGN_OF_DOUBLE PR_ALIGN_OF_DOUBLE -#define ALIGN_OF_POINTER PR_ALIGN_OF_POINTER -#define ALIGN_OF_WORD PR_ALIGN_OF_WORD - -#define BYTES_PER_WORD_LOG2 PR_BYTES_PER_WORD_LOG2 -#define BYTES_PER_DWORD_LOG2 PR_BYTES_PER_DWORD_LOG2 -#define WORDS_PER_DWORD_LOG2 PR_WORDS_PER_DWORD_LOG2 - -#endif /* NO_NSPR_10_SUPPORT */ - -#endif /* nspr_cpucfg___ */ diff --git a/src/ext/libcharsetdetect/nspr-emu/prcpucfg_win.h b/src/ext/libcharsetdetect/nspr-emu/prcpucfg_win.h deleted file mode 100644 index 4ad5345..0000000 --- a/src/ext/libcharsetdetect/nspr-emu/prcpucfg_win.h +++ /dev/null @@ -1,256 +0,0 @@ -/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is the Netscape Portable Runtime (NSPR). - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998-2000 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nspr_cpucfg___ -#define nspr_cpucfg___ - -#ifndef XP_PC -#define XP_PC -#endif - -#ifndef WIN32 -#define WIN32 -#endif - -#ifndef WIN95 -#define WIN95 -#endif - -#define PR_AF_INET6 23 /* same as AF_INET6 */ - -#if defined(_M_IX86) || defined(_X86_) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_WORD 4 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_DOUBLE 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_WORD 32 -#define PR_BITS_PER_DWORD 64 -#define PR_BITS_PER_DOUBLE 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_WORD_LOG2 5 -#define PR_BITS_PER_DWORD_LOG2 6 -#define PR_BITS_PER_DOUBLE_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_WORD 4 -#define PR_ALIGN_OF_DWORD 8 -#define PR_ALIGN_OF_DOUBLE 4 -#define PR_ALIGN_OF_POINTER 4 - -#define PR_BYTES_PER_WORD_LOG2 2 -#define PR_BYTES_PER_DWORD_LOG2 2 - -#elif defined(_M_X64) || defined(_M_AMD64) || defined(_AMD64_) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_DOUBLE 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_WORD 64 -#define PR_BITS_PER_DWORD 64 -#define PR_BITS_PER_DOUBLE 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_WORD_LOG2 6 -#define PR_BITS_PER_DWORD_LOG2 6 -#define PR_BITS_PER_DOUBLE_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_WORD 8 -#define PR_ALIGN_OF_DWORD 8 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 - -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#elif defined(_M_IA64) || defined(_IA64_) - -#define IS_LITTLE_ENDIAN 1 -#undef IS_BIG_ENDIAN -#define IS_64 - -#define PR_BYTES_PER_BYTE 1 -#define PR_BYTES_PER_SHORT 2 -#define PR_BYTES_PER_INT 4 -#define PR_BYTES_PER_INT64 8 -#define PR_BYTES_PER_LONG 4 -#define PR_BYTES_PER_FLOAT 4 -#define PR_BYTES_PER_WORD 8 -#define PR_BYTES_PER_DWORD 8 -#define PR_BYTES_PER_DOUBLE 8 - -#define PR_BITS_PER_BYTE 8 -#define PR_BITS_PER_SHORT 16 -#define PR_BITS_PER_INT 32 -#define PR_BITS_PER_INT64 64 -#define PR_BITS_PER_LONG 32 -#define PR_BITS_PER_FLOAT 32 -#define PR_BITS_PER_WORD 64 -#define PR_BITS_PER_DWORD 64 -#define PR_BITS_PER_DOUBLE 64 - -#define PR_BITS_PER_BYTE_LOG2 3 -#define PR_BITS_PER_SHORT_LOG2 4 -#define PR_BITS_PER_INT_LOG2 5 -#define PR_BITS_PER_INT64_LOG2 6 -#define PR_BITS_PER_LONG_LOG2 5 -#define PR_BITS_PER_FLOAT_LOG2 5 -#define PR_BITS_PER_WORD_LOG2 6 -#define PR_BITS_PER_DWORD_LOG2 6 -#define PR_BITS_PER_DOUBLE_LOG2 6 - -#define PR_ALIGN_OF_SHORT 2 -#define PR_ALIGN_OF_INT 4 -#define PR_ALIGN_OF_LONG 4 -#define PR_ALIGN_OF_INT64 8 -#define PR_ALIGN_OF_FLOAT 4 -#define PR_ALIGN_OF_WORD 8 -#define PR_ALIGN_OF_DWORD 8 -#define PR_ALIGN_OF_DOUBLE 8 -#define PR_ALIGN_OF_POINTER 8 - -#define PR_BYTES_PER_WORD_LOG2 3 -#define PR_BYTES_PER_DWORD_LOG2 3 - -#else /* defined(_M_IX86) || defined(_X86_) */ - -#error unknown processor architecture - -#endif /* defined(_M_IX86) || defined(_X86_) */ - -#ifndef HAVE_LONG_LONG -#define HAVE_LONG_LONG -#endif - -#ifndef NO_NSPR_10_SUPPORT - -#define BYTES_PER_BYTE PR_BYTES_PER_BYTE -#define BYTES_PER_SHORT PR_BYTES_PER_SHORT -#define BYTES_PER_INT PR_BYTES_PER_INT -#define BYTES_PER_INT64 PR_BYTES_PER_INT64 -#define BYTES_PER_LONG PR_BYTES_PER_LONG -#define BYTES_PER_FLOAT PR_BYTES_PER_FLOAT -#define BYTES_PER_DOUBLE PR_BYTES_PER_DOUBLE -#define BYTES_PER_WORD PR_BYTES_PER_WORD -#define BYTES_PER_DWORD PR_BYTES_PER_DWORD - -#define BITS_PER_BYTE PR_BITS_PER_BYTE -#define BITS_PER_SHORT PR_BITS_PER_SHORT -#define BITS_PER_INT PR_BITS_PER_INT -#define BITS_PER_INT64 PR_BITS_PER_INT64 -#define BITS_PER_LONG PR_BITS_PER_LONG -#define BITS_PER_FLOAT PR_BITS_PER_FLOAT -#define BITS_PER_DOUBLE PR_BITS_PER_DOUBLE -#define BITS_PER_WORD PR_BITS_PER_WORD - -#define BITS_PER_BYTE_LOG2 PR_BITS_PER_BYTE_LOG2 -#define BITS_PER_SHORT_LOG2 PR_BITS_PER_SHORT_LOG2 -#define BITS_PER_INT_LOG2 PR_BITS_PER_INT_LOG2 -#define BITS_PER_INT64_LOG2 PR_BITS_PER_INT64_LOG2 -#define BITS_PER_LONG_LOG2 PR_BITS_PER_LONG_LOG2 -#define BITS_PER_FLOAT_LOG2 PR_BITS_PER_FLOAT_LOG2 -#define BITS_PER_DOUBLE_LOG2 PR_BITS_PER_DOUBLE_LOG2 -#define BITS_PER_WORD_LOG2 PR_BITS_PER_WORD_LOG2 - -#define ALIGN_OF_SHORT PR_ALIGN_OF_SHORT -#define ALIGN_OF_INT PR_ALIGN_OF_INT -#define ALIGN_OF_LONG PR_ALIGN_OF_LONG -#define ALIGN_OF_INT64 PR_ALIGN_OF_INT64 -#define ALIGN_OF_FLOAT PR_ALIGN_OF_FLOAT -#define ALIGN_OF_DOUBLE PR_ALIGN_OF_DOUBLE -#define ALIGN_OF_POINTER PR_ALIGN_OF_POINTER -#define ALIGN_OF_WORD PR_ALIGN_OF_WORD - -#define BYTES_PER_WORD_LOG2 PR_BYTES_PER_WORD_LOG2 -#define BYTES_PER_DWORD_LOG2 PR_BYTES_PER_DWORD_LOG2 -#define WORDS_PER_DWORD_LOG2 PR_WORDS_PER_DWORD_LOG2 - -#endif /* NO_NSPR_10_SUPPORT */ - -#endif /* nspr_cpucfg___ */ diff --git a/src/ext/libcharsetdetect/nspr-emu/prmem.h b/src/ext/libcharsetdetect/nspr-emu/prmem.h deleted file mode 100644 index c013ec4..0000000 --- a/src/ext/libcharsetdetect/nspr-emu/prmem.h +++ /dev/null @@ -1,155 +0,0 @@ -/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is the Netscape Portable Runtime (NSPR). - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998-2000 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -/* -** File: prmem.h -** Description: API to NSPR memory management functions -** -*/ -#ifndef prmem_h___ -#define prmem_h___ - -#include "prtypes.h" -#include - -PR_BEGIN_EXTERN_C - -/* -** Thread safe memory allocation. -** -** NOTE: pr wraps up malloc, free, calloc, realloc so they are already -** thread safe (and are not declared here - look in stdlib.h). -*/ - -/* -** PR_Malloc, PR_Calloc, PR_Realloc, and PR_Free have the same signatures -** as their libc equivalent malloc, calloc, realloc, and free, and have -** the same semantics. (Note that the argument type size_t is replaced -** by PRUint32.) Memory allocated by PR_Malloc, PR_Calloc, or PR_Realloc -** must be freed by PR_Free. -*/ - -#define PR_Malloc malloc -#define PR_Calloc calloc -#define PR_Realloc realloc -#define PR_Free free - -/* -** The following are some convenience macros defined in terms of -** PR_Malloc, PR_Calloc, PR_Realloc, and PR_Free. -*/ - -/*********************************************************************** -** FUNCTION: PR_MALLOC() -** DESCRIPTION: -** PR_NEW() allocates an untyped item of size _size from the heap. -** INPUTS: _size: size in bytes of item to be allocated -** OUTPUTS: untyped pointer to the node allocated -** RETURN: pointer to node or error returned from malloc(). -***********************************************************************/ -#define PR_MALLOC(_bytes) (PR_Malloc((_bytes))) - -/*********************************************************************** -** FUNCTION: PR_NEW() -** DESCRIPTION: -** PR_NEW() allocates an item of type _struct from the heap. -** INPUTS: _struct: a data type -** OUTPUTS: pointer to _struct -** RETURN: pointer to _struct or error returns from malloc(). -***********************************************************************/ -#define PR_NEW(_struct) ((_struct *) PR_MALLOC(sizeof(_struct))) - -/*********************************************************************** -** FUNCTION: PR_REALLOC() -** DESCRIPTION: -** PR_REALLOC() re-allocates _ptr bytes from the heap as a _size -** untyped item. -** INPUTS: _ptr: pointer to node to reallocate -** _size: size of node to allocate -** OUTPUTS: pointer to node allocated -** RETURN: pointer to node allocated -***********************************************************************/ -#define PR_REALLOC(_ptr, _size) (PR_Realloc((_ptr), (_size))) - -/*********************************************************************** -** FUNCTION: PR_CALLOC() -** DESCRIPTION: -** PR_CALLOC() allocates a _size bytes untyped item from the heap -** and sets the allocated memory to all 0x00. -** INPUTS: _size: size of node to allocate -** OUTPUTS: pointer to node allocated -** RETURN: pointer to node allocated -***********************************************************************/ -#define PR_CALLOC(_size) (PR_Calloc(1, (_size))) - -/*********************************************************************** -** FUNCTION: PR_NEWZAP() -** DESCRIPTION: -** PR_NEWZAP() allocates an item of type _struct from the heap -** and sets the allocated memory to all 0x00. -** INPUTS: _struct: a data type -** OUTPUTS: pointer to _struct -** RETURN: pointer to _struct -***********************************************************************/ -#define PR_NEWZAP(_struct) ((_struct*)PR_Calloc(1, sizeof(_struct))) - -/*********************************************************************** -** FUNCTION: PR_DELETE() -** DESCRIPTION: -** PR_DELETE() unallocates an object previosly allocated via PR_NEW() -** or PR_NEWZAP() to the heap. -** INPUTS: pointer to previously allocated object -** OUTPUTS: the referenced object is returned to the heap -** RETURN: void -***********************************************************************/ -#define PR_DELETE(_ptr) { PR_Free(_ptr); (_ptr) = NULL; } - -/*********************************************************************** -** FUNCTION: PR_FREEIF() -** DESCRIPTION: -** PR_FREEIF() conditionally unallocates an object previously allocated -** vial PR_NEW() or PR_NEWZAP(). If the pointer to the object is -** equal to zero (0), the object is not released. -** INPUTS: pointer to previously allocated object -** OUTPUTS: the referenced object is conditionally returned to the heap -** RETURN: void -***********************************************************************/ -#define PR_FREEIF(_ptr) if (_ptr) PR_DELETE(_ptr) - -PR_END_EXTERN_C - -#endif /* prmem_h___ */ diff --git a/src/ext/libcharsetdetect/nspr-emu/prtypes.h b/src/ext/libcharsetdetect/nspr-emu/prtypes.h deleted file mode 100644 index b764b27..0000000 --- a/src/ext/libcharsetdetect/nspr-emu/prtypes.h +++ /dev/null @@ -1,533 +0,0 @@ -/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is the Netscape Portable Runtime (NSPR). - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998-2000 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -/* -** File: prtypes.h -** Description: Definitions of NSPR's basic types -** -** Prototypes and macros used to make up for deficiencies that we have found -** in ANSI environments. -** -** Since we do not wrap and all the other standard headers, authors -** of portable code will not know in general that they need these definitions. -** Instead of requiring these authors to find the dependent uses in their code -** and take the following steps only in those C files, we take steps once here -** for all C files. -**/ - -#ifndef prtypes_h___ -#define prtypes_h___ - -#ifdef MDCPUCFG -#include MDCPUCFG -#else -#include "prcpucfg.h" -#endif - -#include - -/*********************************************************************** -** MACROS: PR_EXTERN -** PR_IMPLEMENT -** DESCRIPTION: -** These are only for externally visible routines and globals. For -** internal routines, just use "extern" for type checking and that -** will not export internal cross-file or forward-declared symbols. -** Define a macro for declaring procedures return types. We use this to -** deal with windoze specific type hackery for DLL definitions. Use -** PR_EXTERN when the prototype for the method is declared. Use -** PR_IMPLEMENT for the implementation of the method. -** -** Example: -** in dowhim.h -** PR_EXTERN( void ) DoWhatIMean( void ); -** in dowhim.c -** PR_IMPLEMENT( void ) DoWhatIMean( void ) { return; } -** -** -***********************************************************************/ -#if defined(WIN32) - -#define PR_EXPORT(__type) extern __declspec(dllexport) __type -#define PR_EXPORT_DATA(__type) extern __declspec(dllexport) __type -#define PR_IMPORT(__type) __declspec(dllimport) __type -#define PR_IMPORT_DATA(__type) __declspec(dllimport) __type - -#define PR_EXTERN(__type) extern __declspec(dllexport) __type -#define PR_IMPLEMENT(__type) __declspec(dllexport) __type -#define PR_EXTERN_DATA(__type) extern __declspec(dllexport) __type -#define PR_IMPLEMENT_DATA(__type) __declspec(dllexport) __type - -#define PR_CALLBACK -#define PR_CALLBACK_DECL -#define PR_STATIC_CALLBACK(__x) static __x - -#elif defined(XP_BEOS) - -#define PR_EXPORT(__type) extern __declspec(dllexport) __type -#define PR_EXPORT_DATA(__type) extern __declspec(dllexport) __type -#define PR_IMPORT(__type) extern __declspec(dllexport) __type -#define PR_IMPORT_DATA(__type) extern __declspec(dllexport) __type - -#define PR_EXTERN(__type) extern __declspec(dllexport) __type -#define PR_IMPLEMENT(__type) __declspec(dllexport) __type -#define PR_EXTERN_DATA(__type) extern __declspec(dllexport) __type -#define PR_IMPLEMENT_DATA(__type) __declspec(dllexport) __type - -#define PR_CALLBACK -#define PR_CALLBACK_DECL -#define PR_STATIC_CALLBACK(__x) static __x - -#elif defined(XP_OS2) && defined(__declspec) - -#define PR_EXPORT(__type) extern __declspec(dllexport) __type -#define PR_EXPORT_DATA(__type) extern __declspec(dllexport) __type -#define PR_IMPORT(__type) extern __declspec(dllimport) __type -#define PR_IMPORT_DATA(__type) extern __declspec(dllimport) __type - -#define PR_EXTERN(__type) extern __declspec(dllexport) __type -#define PR_IMPLEMENT(__type) __declspec(dllexport) __type -#define PR_EXTERN_DATA(__type) extern __declspec(dllexport) __type -#define PR_IMPLEMENT_DATA(__type) __declspec(dllexport) __type - -#define PR_CALLBACK -#define PR_CALLBACK_DECL -#define PR_STATIC_CALLBACK(__x) static __x - -#elif defined(SYMBIAN) - -#define PR_EXPORT(__type) extern __declspec(dllexport) __type -#define PR_EXPORT_DATA(__type) extern __declspec(dllexport) __type -#ifdef __WINS__ -#define PR_IMPORT(__type) extern __declspec(dllexport) __type -#define PR_IMPORT_DATA(__type) extern __declspec(dllexport) __type -#else -#define PR_IMPORT(__type) extern __declspec(dllimport) __type -#define PR_IMPORT_DATA(__type) extern __declspec(dllimport) __type -#endif - -#define PR_EXTERN(__type) extern __type -#define PR_IMPLEMENT(__type) __type -#define PR_EXTERN_DATA(__type) extern __type -#define PR_IMPLEMENT_DATA(__type) __type - -#define PR_CALLBACK -#define PR_CALLBACK_DECL -#define PR_STATIC_CALLBACK(__x) static __x - -#else /* Unix */ - -/* GCC 3.3 and later support the visibility attribute. */ -#if (__GNUC__ >= 4) || \ - (__GNUC__ == 3 && __GNUC_MINOR__ >= 3) -#define PR_VISIBILITY_DEFAULT __attribute__((visibility("default"))) -#else -#define PR_VISIBILITY_DEFAULT -#endif - -#define PR_EXPORT(__type) extern PR_VISIBILITY_DEFAULT __type -#define PR_EXPORT_DATA(__type) extern PR_VISIBILITY_DEFAULT __type -#define PR_IMPORT(__type) extern PR_VISIBILITY_DEFAULT __type -#define PR_IMPORT_DATA(__type) extern PR_VISIBILITY_DEFAULT __type - -#define PR_EXTERN(__type) extern PR_VISIBILITY_DEFAULT __type -#define PR_IMPLEMENT(__type) PR_VISIBILITY_DEFAULT __type -#define PR_EXTERN_DATA(__type) extern PR_VISIBILITY_DEFAULT __type -#define PR_IMPLEMENT_DATA(__type) PR_VISIBILITY_DEFAULT __type -#define PR_CALLBACK -#define PR_CALLBACK_DECL -#define PR_STATIC_CALLBACK(__x) static __x - -#endif - -#if defined(_NSPR_BUILD_) -#define NSPR_API(__type) PR_EXPORT(__type) -#define NSPR_DATA_API(__type) PR_EXPORT_DATA(__type) -#else -#define NSPR_API(__type) PR_IMPORT(__type) -#define NSPR_DATA_API(__type) PR_IMPORT_DATA(__type) -#endif - -/*********************************************************************** -** MACROS: PR_BEGIN_MACRO -** PR_END_MACRO -** DESCRIPTION: -** Macro body brackets so that macros with compound statement definitions -** behave syntactically more like functions when called. -***********************************************************************/ -#define PR_BEGIN_MACRO do { -#define PR_END_MACRO } while (0) - -/*********************************************************************** -** MACROS: PR_BEGIN_EXTERN_C -** PR_END_EXTERN_C -** DESCRIPTION: -** Macro shorthands for conditional C++ extern block delimiters. -***********************************************************************/ -#ifdef __cplusplus -#define PR_BEGIN_EXTERN_C extern "C" { -#define PR_END_EXTERN_C } -#else -#define PR_BEGIN_EXTERN_C -#define PR_END_EXTERN_C -#endif - -/*********************************************************************** -** MACROS: PR_BIT -** PR_BITMASK -** DESCRIPTION: -** Bit masking macros. XXX n must be <= 31 to be portable -***********************************************************************/ -#define PR_BIT(n) ((PRUint32)1 << (n)) -#define PR_BITMASK(n) (PR_BIT(n) - 1) - -/*********************************************************************** -** MACROS: PR_ROUNDUP -** PR_MIN -** PR_MAX -** PR_ABS -** DESCRIPTION: -** Commonly used macros for operations on compatible types. -***********************************************************************/ -#define PR_ROUNDUP(x,y) ((((x)+((y)-1))/(y))*(y)) -#define PR_MIN(x,y) ((x)<(y)?(x):(y)) -#define PR_MAX(x,y) ((x)>(y)?(x):(y)) -#define PR_ABS(x) ((x)<0?-(x):(x)) - -PR_BEGIN_EXTERN_C - -/************************************************************************ -** TYPES: PRUint8 -** PRInt8 -** DESCRIPTION: -** The int8 types are known to be 8 bits each. There is no type that -** is equivalent to a plain "char". -************************************************************************/ -#if PR_BYTES_PER_BYTE == 1 -typedef unsigned char PRUint8; -/* -** Some cfront-based C++ compilers do not like 'signed char' and -** issue the warning message: -** warning: "signed" not implemented (ignored) -** For these compilers, we have to define PRInt8 as plain 'char'. -** Make sure that plain 'char' is indeed signed under these compilers. -*/ -#if (defined(HPUX) && defined(__cplusplus) \ - && !defined(__GNUC__) && __cplusplus < 199707L) \ - || (defined(SCO) && defined(__cplusplus) \ - && !defined(__GNUC__) && __cplusplus == 1L) -typedef char PRInt8; -#else -typedef signed char PRInt8; -#endif -#else -#error No suitable type for PRInt8/PRUint8 -#endif - -/************************************************************************ - * MACROS: PR_INT8_MAX - * PR_INT8_MIN - * PR_UINT8_MAX - * DESCRIPTION: - * The maximum and minimum values of a PRInt8 or PRUint8. -************************************************************************/ - -#define PR_INT8_MAX 127 -#define PR_INT8_MIN (-128) -#define PR_UINT8_MAX 255U - -/************************************************************************ -** TYPES: PRUint16 -** PRInt16 -** DESCRIPTION: -** The int16 types are known to be 16 bits each. -************************************************************************/ -#if PR_BYTES_PER_SHORT == 2 -typedef unsigned short PRUint16; -typedef short PRInt16; -#else -#error No suitable type for PRInt16/PRUint16 -#endif - -/************************************************************************ - * MACROS: PR_INT16_MAX - * PR_INT16_MIN - * PR_UINT16_MAX - * DESCRIPTION: - * The maximum and minimum values of a PRInt16 or PRUint16. -************************************************************************/ - -#define PR_INT16_MAX 32767 -#define PR_INT16_MIN (-32768) -#define PR_UINT16_MAX 65535U - -/************************************************************************ -** TYPES: PRUint32 -** PRInt32 -** DESCRIPTION: -** The int32 types are known to be 32 bits each. -************************************************************************/ -#if PR_BYTES_PER_INT == 4 -typedef unsigned int PRUint32; -typedef int PRInt32; -#define PR_INT32(x) x -#define PR_UINT32(x) x ## U -#elif PR_BYTES_PER_LONG == 4 -typedef unsigned long PRUint32; -typedef long PRInt32; -#define PR_INT32(x) x ## L -#define PR_UINT32(x) x ## UL -#else -#error No suitable type for PRInt32/PRUint32 -#endif - -/************************************************************************ - * MACROS: PR_INT32_MAX - * PR_INT32_MIN - * PR_UINT32_MAX - * DESCRIPTION: - * The maximum and minimum values of a PRInt32 or PRUint32. -************************************************************************/ - -#define PR_INT32_MAX PR_INT32(2147483647) -#define PR_INT32_MIN (-PR_INT32_MAX - 1) -#define PR_UINT32_MAX PR_UINT32(4294967295) - -/************************************************************************ -** TYPES: PRUint64 -** PRInt64 -** DESCRIPTION: -** The int64 types are known to be 64 bits each. Care must be used when -** declaring variables of type PRUint64 or PRInt64. Different hardware -** architectures and even different compilers have varying support for -** 64 bit values. The only guaranteed portability requires the use of -** the LL_ macros (see prlong.h). -************************************************************************/ -#ifdef HAVE_LONG_LONG -/* Keep this in sync with prlong.h. */ -/* - * On 64-bit Mac OS X, uint64 needs to be defined as unsigned long long to - * match uint64_t, otherwise our uint64 typedef conflicts with the uint64 - * typedef in cssmconfig.h, which CoreServices.h includes indirectly. - */ -#if PR_BYTES_PER_LONG == 8 && !defined(__APPLE__) -typedef long PRInt64; -typedef unsigned long PRUint64; -#elif defined(WIN32) && !defined(__GNUC__) -typedef __int64 PRInt64; -typedef unsigned __int64 PRUint64; -#else -typedef long long PRInt64; -typedef unsigned long long PRUint64; -#endif /* PR_BYTES_PER_LONG == 8 */ -#else /* !HAVE_LONG_LONG */ -typedef struct { -#ifdef IS_LITTLE_ENDIAN - PRUint32 lo, hi; -#else - PRUint32 hi, lo; -#endif -} PRInt64; -typedef PRInt64 PRUint64; -#endif /* !HAVE_LONG_LONG */ - -/************************************************************************ -** TYPES: PRUintn -** PRIntn -** DESCRIPTION: -** The PRIntn types are most appropriate for automatic variables. They are -** guaranteed to be at least 16 bits, though various architectures may -** define them to be wider (e.g., 32 or even 64 bits). These types are -** never valid for fields of a structure. -************************************************************************/ -#if PR_BYTES_PER_INT >= 2 -typedef int PRIntn; -typedef unsigned int PRUintn; -#else -#error 'sizeof(int)' not sufficient for platform use -#endif - -/************************************************************************ -** TYPES: PRFloat64 -** DESCRIPTION: -** NSPR's floating point type is always 64 bits. -************************************************************************/ -typedef double PRFloat64; - -/************************************************************************ -** TYPES: PRSize -** DESCRIPTION: -** A type for representing the size of objects. -************************************************************************/ -typedef size_t PRSize; - - -/************************************************************************ -** TYPES: PROffset32, PROffset64 -** DESCRIPTION: -** A type for representing byte offsets from some location. -************************************************************************/ -typedef PRInt32 PROffset32; -typedef PRInt64 PROffset64; - -/************************************************************************ -** TYPES: PRPtrDiff -** DESCRIPTION: -** A type for pointer difference. Variables of this type are suitable -** for storing a pointer or pointer subtraction. -************************************************************************/ -typedef ptrdiff_t PRPtrdiff; - -/************************************************************************ -** TYPES: PRUptrdiff -** DESCRIPTION: -** A type for pointer difference. Variables of this type are suitable -** for storing a pointer or pointer sutraction. -************************************************************************/ -#ifdef _WIN64 -typedef PRUint64 PRUptrdiff; -#else -typedef unsigned long PRUptrdiff; -#endif - -/************************************************************************ -** TYPES: PRBool -** DESCRIPTION: -** Use PRBool for variables and parameter types. Use PR_FALSE and PR_TRUE -** for clarity of target type in assignments and actual arguments. Use -** 'if (bool)', 'while (!bool)', '(bool) ? x : y' etc., to test booleans -** just as you would C int-valued conditions. -************************************************************************/ -typedef PRIntn PRBool; -#define PR_TRUE 1 -#define PR_FALSE 0 - -/************************************************************************ -** TYPES: PRPackedBool -** DESCRIPTION: -** Use PRPackedBool within structs where bitfields are not desirable -** but minimum and consistant overhead matters. -************************************************************************/ -typedef PRUint8 PRPackedBool; - -/* -** Status code used by some routines that have a single point of failure or -** special status return. -*/ -typedef enum { PR_FAILURE = -1, PR_SUCCESS = 0 } PRStatus; - -#ifndef __PRUNICHAR__ -#define __PRUNICHAR__ -#ifdef WIN32 -typedef wchar_t PRUnichar; -#else -typedef PRUint16 PRUnichar; -#endif -#endif - -/* -** WARNING: The undocumented data types PRWord and PRUword are -** only used in the garbage collection and arena code. Do not -** use PRWord and PRUword in new code. -** -** A PRWord is an integer that is the same size as a void*. -** It implements the notion of a "word" in the Java Virtual -** Machine. (See Sec. 3.4 "Words", The Java Virtual Machine -** Specification, Addison-Wesley, September 1996. -** http://java.sun.com/docs/books/vmspec/index.html.) -*/ -#ifdef _WIN64 -typedef PRInt64 PRWord; -typedef PRUint64 PRUword; -#else -typedef long PRWord; -typedef unsigned long PRUword; -#endif - -#if defined(NO_NSPR_10_SUPPORT) -#else -/********* ???????????????? FIX ME ??????????????????????????? *****/ -/********************** Some old definitions until pr=>ds transition is done ***/ -/********************** Also, we are still using NSPR 1.0. GC ******************/ -/* -** Fundamental NSPR macros, used nearly everywhere. -*/ - -#define PR_PUBLIC_API PR_IMPLEMENT - -/* -** Macro body brackets so that macros with compound statement definitions -** behave syntactically more like functions when called. -*/ -#define NSPR_BEGIN_MACRO do { -#define NSPR_END_MACRO } while (0) - -/* -** Macro shorthands for conditional C++ extern block delimiters. -*/ -#ifdef NSPR_BEGIN_EXTERN_C -#undef NSPR_BEGIN_EXTERN_C -#endif -#ifdef NSPR_END_EXTERN_C -#undef NSPR_END_EXTERN_C -#endif - -#ifdef __cplusplus -#define NSPR_BEGIN_EXTERN_C extern "C" { -#define NSPR_END_EXTERN_C } -#else -#define NSPR_BEGIN_EXTERN_C -#define NSPR_END_EXTERN_C -#endif - -#include "obsolete/protypes.h" - -/********* ????????????? End Fix me ?????????????????????????????? *****/ -#endif /* NO_NSPR_10_SUPPORT */ - -/* -** Compile-time assert. "condition" must be a constant expression. -** The macro can be used only in places where an "extern" declaration is -** allowed. -*/ -#define PR_STATIC_ASSERT(condition) \ - extern void pr_static_assert(int arg[(condition) ? 1 : -1]) - -PR_END_EXTERN_C - -#endif /* prtypes_h___ */ - diff --git a/src/ext/libcharsetdetect/trchardet.c b/src/ext/libcharsetdetect/trchardet.c deleted file mode 100644 index aba0b97..0000000 --- a/src/ext/libcharsetdetect/trchardet.c +++ /dev/null @@ -1,38 +0,0 @@ -#include "charsetdetect.h" -#include "stdio.h" - -#define BUFFER_SIZE 100*1024 - -int main(int argc, const char * argv[]) { - csd_t csd = csd_open(); - if (csd == (csd_t)-1) { - printf("csd_open failed\n"); - return 1; - } - - int size; - char buf[BUFFER_SIZE] = {0}; - - while ((size = fread(buf, 1, sizeof(buf), stdin)) != 0) { - printf("CLIENT SENDING More data\n"); - int result = csd_consider(csd, buf, size); - if (result < 0) { - printf("csd_consider failed\n"); - return 3; - } else if (result == 0) { - // Already have enough data - break; - } - // Only send one buffer actually, for testing - break; - } - - const char *result = csd_close(csd); - if (result == NULL) { - printf("Unknown character set\n"); - return 2; - } else { - printf("%s\n", result); - return 0; - } -} diff --git a/src/ext/libcharsetdetect/update-mozilla b/src/ext/libcharsetdetect/update-mozilla deleted file mode 100644 index 4d71523..0000000 --- a/src/ext/libcharsetdetect/update-mozilla +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -cvs -d :pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot co mozilla/extensions/universalchardet/src/base diff --git a/src/ext/uchardet b/src/ext/uchardet new file mode 160000 index 0000000..795e982 --- /dev/null +++ b/src/ext/uchardet @@ -0,0 +1 @@ +Subproject commit 795e982965404568e3a8354e69b08e71137ee6ff diff --git a/src/tests/bench.py b/src/tests/bench.py index 02b9667..2b23730 100644 --- a/src/tests/bench.py +++ b/src/tests/bench.py @@ -6,7 +6,7 @@ import chardet def main(): do_times = 100 - path = r'tests/testdata/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt' + path = r'tests/samples/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt' with open(path, 'rb') as f: msg = f.read() diff --git a/src/tests/testdata/wikipediaJa_Japan_SJIS.txt b/src/tests/samples/wikipediaJa_Japan_SJIS.txt similarity index 100% rename from src/tests/testdata/wikipediaJa_Japan_SJIS.txt rename to src/tests/samples/wikipediaJa_Japan_SJIS.txt diff --git a/src/tests/testdata/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt b/src/tests/samples/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt similarity index 100% rename from src/tests/testdata/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt rename to src/tests/samples/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt diff --git a/src/tests/testdata/wikipediaTw_Japan_GB2312.txt b/src/tests/samples/wikipediaTw_Japan_GB2312.txt similarity index 100% rename from src/tests/testdata/wikipediaTw_Japan_GB2312.txt rename to src/tests/samples/wikipediaTw_Japan_GB2312.txt diff --git a/src/tests/testdata/wikipediaTw_One_Thousand_and_One_Nights_GB2312.txt b/src/tests/samples/wikipediaTw_One_Thousand_and_One_Nights_GB2312.txt similarity index 100% rename from src/tests/testdata/wikipediaTw_One_Thousand_and_One_Nights_GB2312.txt rename to src/tests/samples/wikipediaTw_One_Thousand_and_One_Nights_GB2312.txt diff --git a/src/tests/test.py b/src/tests/test.py index 85df699..88919b7 100644 --- a/src/tests/test.py +++ b/src/tests/test.py @@ -1,88 +1,104 @@ import sys +import glob +import os +import string from nose.tools import eq_ - import cchardet -encodings_map = { - r'tests/testdata/bg/ISO-8859-5/wikitop_bg_ISO-8859-5.txt': 'ISO-8859-5', - r'tests/testdata/bg/UTF-8/wikitop_bg_UTF-8.txt': 'UTF-8', - r'tests/testdata/bg/WINDOWS-1251/wikitop_bg_WINDOWS-1251.txt': 'WINDOWS-1251', - r'tests/testdata/cz/ISO-8859-2/wikitop_cz_ISO-8859-2.txt': 'ISO-8859-2', - r'tests/testdata/cz/UTF-8/wikitop_cz_UTF-8.txt': 'UTF-8', - r'tests/testdata/de/UTF-8/wikitop_de_UTF-8.txt': 'UTF-8', - r'tests/testdata/de/WINDOWS-1252/wikitop_de_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/dk/UTF-8/wikitop_dk_UTF-8.txt': 'UTF-8', - r'tests/testdata/dk/WINDOWS-1252/wikitop_dk_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/el/ISO-8859-7/wikitop_el_ISO-8859-7.txt': 'ISO-8859-7', - r'tests/testdata/el/UTF-8/wikitop_el_UTF-8.txt': 'UTF-8', - r'tests/testdata/en/UTF-8/wikitop_en_UTF-8.txt': 'UTF-8', - r'tests/testdata/en/WINDOWS-1252/wikitop_en_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/es/UTF-8/wikitop_es_UTF-8.txt': 'UTF-8', - r'tests/testdata/es/WINDOWS-1252/wikitop_es_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/fi/UTF-8/wikitop_fi_UTF-8.txt': 'UTF-8', - r'tests/testdata/fi/WINDOWS-1252/wikitop_fi_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/fr/UTF-8/wikitop_fr_UTF-8.txt': 'UTF-8', - r'tests/testdata/fr/WINDOWS-1252/wikitop_fr_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/he/UTF-8/wikitop_he_UTF-8.txt': 'UTF-8', - r'tests/testdata/he/WINDOWS-1255/wikitop_he_WINDOWS-1255.txt': 'WINDOWS-1255', - r'tests/testdata/hu/UTF-8/wikitop_hu_UTF-8.txt': 'UTF-8', - r'tests/testdata/hu/ISO-8859-2/wikitop_hu_ISO-8859-2.txt': 'ISO-8859-2', - r'tests/testdata/it/UTF-8/wikitop_it_UTF-8.txt': 'UTF-8', - r'tests/testdata/it/WINDOWS-1252/wikitop_it_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/nl/UTF-8/wikitop_nl_UTF-8.txt': 'UTF-8', - r'tests/testdata/nl/WINDOWS-1252/wikitop_nl_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/no/UTF-8/wikitop_no_UTF-8.txt': 'UTF-8', - r'tests/testdata/no/WINDOWS-1252/wikitop_no_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/pl/UTF-8/wikitop_pl_UTF-8.txt': 'UTF-8', - r'tests/testdata/pl/ISO-8859-2/wikitop_pl_ISO-8859-2.txt': 'ISO-8859-2', - r'tests/testdata/pt/UTF-8/wikitop_pt_UTF-8.txt': 'UTF-8', - r'tests/testdata/pt/WINDOWS-1252/wikitop_pt_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/ru/UTF-8/wikitop_ru_UTF-8.txt': 'UTF-8', - r'tests/testdata/ru/WINDOWS-1251/wikitop_ru_WINDOWS-1251.txt': 'WINDOWS-1251', - r'tests/testdata/ru/IBM855/wikitop_ru_IBM855.txt': 'IBM855', - r'tests/testdata/ru/KOI8-R/wikitop_ru_KOI8-R.txt': 'KOI8-R', - r'tests/testdata/ru/X-MAC-CYRILLIC/wikitop_ru_MACCYRILLIC.txt': 'MAC-CYRILLIC', - r'tests/testdata/se/UTF-8/wikitop_se_UTF-8.txt': 'UTF-8', - r'tests/testdata/se/WINDOWS-1252/wikitop_se_WINDOWS-1252.txt': 'WINDOWS-1252', - r'tests/testdata/th/UTF-8/wikitop_th_UTF-8.txt': 'UTF-8', - r'tests/testdata/th/TIS-620/utffool_th_TIS-620.txt': 'TIS-620', - r'tests/testdata/th/TIS-620/wikitop_th_TIS-620.txt': 'TIS-620', - r'tests/testdata/tr/UTF-8/wikitop_tr_UTF-8.txt': 'UTF-8', - r'tests/testdata/tr/ISO-8859-9/wikitop_tr_ISO-8859-9.txt': 'ISO-8859-9', - r'tests/testdata/zh/UTF-8/wikitop_zh_UTF-8.txt': 'UTF-8', - r'tests/testdata/zh/GB18030/wikitop_zh_GB18030.txt': 'GB18030', -} +SKIP_LIST = [ + 'tests/testdata/ja/utf-16le.txt', + 'tests/testdata/ja/utf-16be.txt', + 'tests/testdata/es/iso-8859-15.txt', + 'tests/testdata/da/iso-8859-1.txt', + 'tests/testdata/he/iso-8859-8.txt' +] +# Python can't decode encoding +SKIP_LIST_02 = [ + 'tests/testdata/vi/viscii.txt', + 'tests/testdata/zh/euc-tw.txt' +] +SKIP_LIST_02.extend(SKIP_LIST) class TestCChardet(): + def test_ascii(self): + detected_encoding = cchardet.detect(b'abcdefghijklmnopqrstuvwxyz') + eq_( + 'ascii', + detected_encoding['encoding'].lower(), + 'Expected %s, but got %s' % ( + 'ascii', + detected_encoding['encoding'].lower() + ) + ) + def test_detect(self): - for path, encoding in encodings_map.items(): - with open(path, 'rb') as f: + testfiles = glob.glob('tests/testdata/*/*.txt') + for testfile in testfiles: + if testfile.replace("\\", "/") in SKIP_LIST: + continue + + base = os.path.basename(testfile) + expected_charset = os.path.splitext(base)[0] + with open(testfile, 'rb') as f: msg = f.read() detected_encoding = cchardet.detect(msg) - eq_(encoding.lower(), detected_encoding['encoding'].lower(), 'Invalid encoding: %s' % path) + eq_( + expected_charset.lower(), + detected_encoding['encoding'].lower(), + 'Expected %s, but got %s for "%s"' % ( + expected_charset.lower(), + detected_encoding['encoding'].lower(), + testfile + ) + ) def test_detector(self): - for path, encoding in encodings_map.items(): - detector = cchardet.Detector() - with open(path, 'rb') as f: + detector = cchardet.UniversalDetector() + with open("tests/samples/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt", 'rb') as f: + line = f.readline() + while line: + detector.feed(line) + if detector.done: + break line = f.readline() - while line: - detector.feed(line) - if detector.done: - break - line = f.readline() - detector.close() - detected_encoding = detector.result - eq_(encoding.lower(), detected_encoding['encoding'].lower(), 'Invalid encoding: %s' % path) + detector.close() + detected_encoding = detector.result + eq_( + "shift_jis", + detected_encoding['encoding'].lower(), + 'Expected %s, but got %s' % ( + "shift_jis", + detected_encoding['encoding'].lower() + ) + ) + + def test_github_issue_20(self): + """ + https://github.com/PyYoshi/cChardet/issues/20 + """ + msg = b'\x8f' - def test_detector_noresult(self): - detector = cchardet.Detector() - if sys.version_info[0] < 3: - zero = '0' - else: - zero = b'0' - detector.feed(zero) - eq_(detector.done, False) - eq_(detector.result, {'encoding': None, 'confidence': None}) + cchardet.detect(msg) + + detector = cchardet.UniversalDetector() + detector.feed(msg) + detector.close() + + def test_decode(self): + testfiles = glob.glob('tests/testdata/*/*.txt') + for testfile in testfiles: + if testfile.replace("\\", "/") in SKIP_LIST_02: + continue + + base = os.path.basename(testfile) + expected_charset = os.path.splitext(base)[0] + with open(testfile, 'rb') as f: + msg = f.read() + detected_encoding = cchardet.detect(msg) + try: + msg.decode(detected_encoding["encoding"]) + except LookupError as e: + print("LookupError: { file=%s, encoding=%s }" % (testfile, detected_encoding["encoding"])) + raise e diff --git a/src/tests/testdata/ar/iso-8859-6.txt b/src/tests/testdata/ar/iso-8859-6.txt new file mode 100644 index 0000000..4831b6a --- /dev/null +++ b/src/tests/testdata/ar/iso-8859-6.txt @@ -0,0 +1,3 @@ +-1256 + . +. 8859-6. diff --git a/src/tests/testdata/ar/utf-8.txt b/src/tests/testdata/ar/utf-8.txt new file mode 100644 index 0000000..37e9a11 --- /dev/null +++ b/src/tests/testdata/ar/utf-8.txt @@ -0,0 +1,3 @@ +ويندوز-1256 هي صفحة كود تستخدم في كتابة اللغة العربية عموماً وبعض اللغات الشبيهة +التي تستخدم نفس الأبجدية مثل الأردو والفارسية والكوردية. وذلك تحت نظام مايكروسوفت +ويندوز. صفحة الكود هذه لا تتوافق مع الأيزو 8859-6. diff --git a/src/tests/testdata/ar/windows-1256.txt b/src/tests/testdata/ar/windows-1256.txt new file mode 100644 index 0000000..15e257b --- /dev/null +++ b/src/tests/testdata/ar/windows-1256.txt @@ -0,0 +1,3 @@ +-1256 + . +. 8859-6. diff --git a/src/tests/testdata/bg/ISO-8859-5/wikitop_bg_ISO-8859-5.txt b/src/tests/testdata/bg/ISO-8859-5/wikitop_bg_ISO-8859-5.txt deleted file mode 100644 index b5df166..0000000 --- a/src/tests/testdata/bg/ISO-8859-5/wikitop_bg_ISO-8859-5.txt +++ /dev/null @@ -1,246 +0,0 @@ - - - , - : , - - ! - - , - 250 , , . - , - ! - - - 2003 ., 122 482 , - - - - . , - ! - - - . , , - . - - : - - - - - - - , - () - - 24 ... - - - * 1648 . - , - . - * 1877 . - - (1877-1878): - , . - * 1929 . - : - ( ) " ", - . - * 1945 . - . - * 1960 . - , - -16, 90 - . - - ... - - ... - - : - * - - ? - * ? - * - ? - * ? - * ? - * ? - * ? - * - - ? - * - - ? - * - ? - * - - ? - * - - ? - * - ? - * ? - * ? - * - ? - * ? - * - ? - * - ? - * , - - ? - * ? - * 81 , - ? - * - ? - * ? - * ? - - - - - - - - ( : Jonas Brothers) - . - , . - . 2008 - , 2010 - , 2: . - - 21 . - - : It's About Time, Jonas - Brothers, A Little Bit Longer Lines, Vines and Trying Times. - 2008 ., 51- , - - , - . 2009 ., - , - . 2009 . - - 2009- - : 3D - ( - ), - . - - - - - - . - -Architecture see also.svg - - - -Nuvola apps package graphics.svg - - - - - - -Sciences exactes.svg - - - - - - -Nuvola apps kpovmodeler.svg - - - - - -Nuvola apps personal unisex.png - - - - -Terrestrial globe.svg - - - - - - -Nuvola apps kuser.svg - - () - - - - - -Nuvola apps kcmsystem.svg - - - o - - - - - - - - If you don't speak Bulgarian, please visit our embassy. - - - - -, , - -, - - - - - - - . - - . , - - . - - - - - * - - - - * - * - - - - - - * - * - * - - - - - - ____________________ (Submit) - - - - * - * - * - * - * - * - * - - - - * - * - * - * - * - * - * - - - - * : 20:16, 27 2011. - * - 3.0; - . . - - * - * - * diff --git a/src/tests/testdata/bg/UTF-8/wikitop_bg_UTF-8.txt b/src/tests/testdata/bg/UTF-8/wikitop_bg_UTF-8.txt deleted file mode 100644 index fdf07cc..0000000 --- a/src/tests/testdata/bg/UTF-8/wikitop_bg_UTF-8.txt +++ /dev/null @@ -1,248 +0,0 @@ -Начална страница - - от Уикипедия, свободната енциклопедия - Направо към: навигация, търсене - -Добре дошли! - - Wikipedia-logo.png - - Здравейте и добре дошли в Уикипедия, свободната енциклопедия на повече - от 250 езика, която всеки може да редактира, развие и обогати. Ако не - знаете откъде да започнете, препоръчваме ви да прочетете страницата - „Първи стъпки“ и Наръчника на уикипедианеца! - - Първата статия в Уикипедия на български език е създадена в началото на - декември 2003 г., а в момента се работи по 122 482 статии, защитени с - лиценза Криейтив Комънс - Признание - Споделяне на споделеното и - Лиценза за свободна документация на ГНУ. Това означава, че те са - свободни и винаги ще бъдат такива! - - Допълнителна информация и полезни указания ще откриете и в портала на - общността. Като начало, ако желаете, можете да разширите статията за - родния си град или село. - - Портали: Африка · България · География · Европа · Езикознание · - Изкуство · История · Личности · Македония · Математика · Медицина · - Философия · Текущи събития - -Избрана картинка на деня в Общомедия - - Teide qtl1.jpg - - Вулканът Тейде на Канарските острови, Испания - Още избрани картинки (описания) - -На 24 октомври през... - - Сградата на Нюйоркската фондова борса - * 1648 г. - Подписан е Вестфалския мирен договор, който слага край на - Тридесетгодишната война. - * 1877 г. - Руско-турска война (1877-1878): след Битката при Горни - Дъбник, Плевен е напълно обсаден от руската армия. - * 1929 г. - Голямата депресия: Индексите на Нюйоркската фондова борса - (на снимката) се сриват по време на "черния четвъртък", което - довежда до серия от банкрути и предизвиква световна рецесия. - * 1945 г. - Приет е уставът на Организацията на обединените нации. - * 1960 г. - При взрив на космодрума Байконур, възникнал при - изпитването на съветската балистична ракета Р-16, загиват около 90 - души. - - Всички дати – Още събития... - -Знаете ли... - - Из новите статии в Уикипедия: - * … кой е най-новият историко-археологически резерват в България? - * … кой е византийският посланик в двора на Атила? - * … че на следващото световно първенство по шахмат за пръв път ще - играе състезател от Израел? - * … кой самоук британски офицер основава империята Британска Индия? - * … кой торбест бозайник не притежава кожна торба? - * … кой е единственият морски пейзаж на Рембранд? - * … коя е първата обсада от Стогодишната война? - * … кое е мястото в света известно с най-висока концентрация на - албинизъм сред хората? - * … кое е мястото в Европа обитавано от най-големия в света табун - коне на Пржевалски? - * … кога е разформирована службата за защита на населението на - Република България? - * … представителите на кое семейство бозайници са с най-кратка - бременност? - * … коя екзопланета има най-голям потенциал за съществуване на живот - засега? - * … коя е най-близката по размери до Земята екзопланета досега? - * … в рамките на кой национален парк се намира връх Еверест? - * … колко вида емпирични формули има в химията? - * … кое е най-студеното населено място на Земята? - * … коя е столицата на Дакийското царство? - * … че най-западното село в Румъния е основано от банатски българи? - * … кой е най-близкият до Великобритания френски град? - * … кое е едно от малкото места в Европа, което е обитавано от трите - най-големи хищника на континента? - * … как се наричат първите печатни книги? - * … кой германски физик е бил номиниран 81 пъти за Нобелова награда, - но не я е получил нито веднъж? - * … коя е най-старата сграда в румънския град Тимишоара? - * … кой турски революционер е войвода на ВМРО? - * … кой български революционер е обявен за „Праведник на света“? - - Архив - -Избрана статия на седмицата - - Джонас Брадърс - - Джонас Брадърс (на английски: Jonas Brothers) е американска момчешка - поп група. Оригиналният и засега постоянен състав на групата включва - тримата братя Кевин Джонас, Джо Джонас и Ник Джонас. Те са от Ню Джързи - и стават популярни чрез детския канал Дисни. През лятото на 2008 - участват в оригиналния филм на Дисни — Кемп Рок, а през 2010 и в - продължението му, Кемп Рок 2: Последният концерт. Това е една от - най-популярните тийнейджърски групи за първото десетилетие на 21 век. - - До момента групата има издадени четири албума: It's About Time, Jonas - Brothers, A Little Bit Longer и Lines, Vines and Trying Times. През - 2008 г., на 51-те Награди Грами, е номинирана за „Най-добър млад - изпълнител“, а на Американските музикални награди групата печели - наградата за „Дебютиращ изпълнител“. До май 2009 г., преди излизането - на четвъртия им албум, имат над осем милиона продадени албума по цял - свят. През 2009 г. братята са избрани категорично и единодушно за - най-лоши актьори на 2009-а във филма-концерт Джонас Брадърс: 3D - концертът на наградите „Златна малинка“ (контрапунктът на наградите - „Оскар“), обявен и за един от най-лошите филми на годината. още» - - Още избрани статии - -Съдържание - - Това е общ преглед на категориите в Уикипедия по теми. - -Architecture see also.svg История - - Историография — По епохи — По страни — По тематика — Хроники - -Nuvola apps package graphics.svg ‎ Култура - - Анимация — Актьори — Езици — Забавления — Знаменитости — Играчки — - Игри — Изкуство — Литература — Световна култура — Медии — Митология — - Музеи — Музика — Празници — Радио — Религия — Спорт — Танци — - Телевизия — Традиции — Туризъм — Фестивали — Философия — Хумор - -Sciences exactes.svg Наука - - Антропология — Археология — Астрономия — Биология — Космос — Науки за - Земята — Информатика — История на науката — Медицина — Политика — - Приложни науки — Природни бедствия — Психология — Социология — Учени — - Физика — Философия на науката — Химия — Екология - -Nuvola apps kpovmodeler.svg Математика - - Алгебра — Анализ — Аритметика — Геометрия — Измерване — Логика — - Приложна математика — Статистика — Теория на вероятностите — Теория на - числата — Теореми — Тригонометрия — Числа — Икономика - -Nuvola apps personal unisex.png ‎ Човек - - Образование — Семейство — Хранене — Празници — Здраве — Хора — - Домашни животни - -Terrestrial globe.svg География - - Африка — Антарктида — Азия — Австралия — Европа — Северна Америка — - Океания — Южна Америка - Градове — Държави — Релеф — Климат — Национални паркове — Селища — - Сгради — Недра - -Nuvola apps kuser.svg Общество - - Хуманитарни (Обществени) науки — Антропология — Археология — - Автоматизация — Бизнес — Комуникация — Демография — Икономика — - Народи — Финанси — Държавно управление — История — Промишленост — - Право — Лингвистика — Медии — Организации — Философия — Политология — - Психология — Социология — Сексология — Войни - -Nuvola apps kcmsystem.svg Техника и технологии - - Селско стопанство — Архитектура — Биотехнология — Военни технологии — - Електроника — Машинoстроене — Интернет — Информационни технологии — - Медицина — Нанотехнология — Ядрена техника — Телекомуникации — - Инструменти — Транспорт — Средства за придвижване — Енергетика - -По азбучен ред - - А Б В Г Д Е Ж З И Й К Л М Н О П Р С Т У Ф Х Ц Ч Ш Щ Ъ Ь Ю Я - - If you don't speak Bulgarian, please visit our embassy. - -Сродни проекти - - Уикицитат Уикицитат -Цитати, афоризми, крилати фрази Уикиречник Уикиречник - Многоезичен речник Уикикниги Уикикниги -Книги, учебници и ръководства Уикиновини Уикиновини - Новини - Общомедия Общомедия -Свободно хранилище Уикиизточник Уикиизточник -Свободни източници Уикивидове Уикивидове - Регистър на видовете Метауики Метауики - Съгласуване на проектите -Сродните на Уикипедия проекти се поддържат от Фондация Уикимедия. - - Тази страница е защитена. Ако желаете да направите промени, моля - редактирайте тестовата страница или ни оставете бележка на нейната - дискусионна страница. - Взето от - -Лични инструменти - - * Регистриране или влизане - -Именни пространства - - * Статия - * Беседа - -Варианти - -Прегледи - - * Преглед - * Преглед на кода - * История - -Действия - -Търсене - - ____________________ (Submit) Търсене - -Навигация - - * Начална страница - * Портал на общността - * Съдържание - * Последни промени - * Случайна статия - * Помощ - * Дарения - -Инструменти - - * Какво сочи насам - * Свързани промени - * Качване в Общомедия - * Специални страници - * Версия за печат - * Постоянна препратка - * Цитиране на статията - -На други езици - * Последна промяна на страницата: 20:16, 27 януари 2011. - * Текстът е достъпен при условията на лиценза Криейтив Комънс - Признание — Споделяне на споделеното 3.0; в сила може да има и - допълнителни условия. За подробности вижте Условията за ползване. - - * Защита на личните данни - * За Уикипедия - * Условия за ползване diff --git a/src/tests/testdata/bg/WINDOWS-1251/wikitop_bg_WINDOWS-1251.txt b/src/tests/testdata/bg/WINDOWS-1251/wikitop_bg_WINDOWS-1251.txt deleted file mode 100644 index b37a0af..0000000 --- a/src/tests/testdata/bg/WINDOWS-1251/wikitop_bg_WINDOWS-1251.txt +++ /dev/null @@ -1,248 +0,0 @@ - - - , - : , - - ! - - Wikipedia-logo.png - - , - 250 , , . - , - ! - - - 2003 ., 122 482 , - - - - . , - ! - - - . , , - . - - : - - - - - - Teide qtl1.jpg - - , - () - - 24 ... - - - * 1648 . - , - . - * 1877 . - - (1877-1878): - , . - * 1929 . - : - ( ) " ", - . - * 1945 . - . - * 1960 . - , - -16, 90 - . - - ... - - ... - - : - * - - ? - * ? - * - ? - * ? - * ? - * ? - * ? - * - - ? - * - - ? - * - ? - * - - ? - * - - ? - * - ? - * ? - * ? - * - ? - * ? - * - ? - * - ? - * , - - ? - * ? - * 81 , - ? - * - ? - * ? - * ? - - - - - - - - ( : Jonas Brothers) - . - , . - . 2008 - , 2010 - , 2: . - - 21 . - - : It's About Time, Jonas - Brothers, A Little Bit Longer Lines, Vines and Trying Times. - 2008 ., 51- , - - , - . 2009 ., - , - . 2009 . - - 2009- - : 3D - ( - ), - . - - - - - - . - -Architecture see also.svg - - - -Nuvola apps package graphics.svg - - - - - - -Sciences exactes.svg - - - - - - -Nuvola apps kpovmodeler.svg - - - - - -Nuvola apps personal unisex.png - - - - -Terrestrial globe.svg - - - - - - -Nuvola apps kuser.svg - - () - - - - - -Nuvola apps kcmsystem.svg - - - o - - - - - - - - If you don't speak Bulgarian, please visit our embassy. - - - - -, , - -, - - - - - - - . - - . , - - . - - - - - * - - - - * - * - - - - - - * - * - * - - - - - - ____________________ (Submit) - - - - * - * - * - * - * - * - * - - - - * - * - * - * - * - * - * - - - * : 20:16, 27 2011. - * - 3.0; - . . - - * - * - * diff --git a/src/tests/testdata/bg/windows-1251.txt b/src/tests/testdata/bg/windows-1251.txt new file mode 100644 index 0000000..550b0a8 --- /dev/null +++ b/src/tests/testdata/bg/windows-1251.txt @@ -0,0 +1,3 @@ +Windows-1251 8- () , , , . + +Windows-1251 KOI8-R ( KOI8-U) - ISO 8859-5, . , . diff --git a/src/tests/testdata/cs/ibm852.txt b/src/tests/testdata/cs/ibm852.txt new file mode 100644 index 0000000..db9f094 --- /dev/null +++ b/src/tests/testdata/cs/ibm852.txt @@ -0,0 +1,4 @@ +Led堟ek n (Alcedo atthis) je prmrn 16,5 cm velk ptk z eledi +led堟kovitch (Alcedinidae). Je velmi vrazn zbarven s oranovou spodinou a +modrm hbetem, kdly a temenem. Vraznm znakem je tak jeho npadn dlouh +zapiatl zobk. Pro sv krsn zbarven je nazvn Ltajc drahokam. diff --git a/src/tests/testdata/cs/iso-8859-2.txt b/src/tests/testdata/cs/iso-8859-2.txt new file mode 100644 index 0000000..95976c4 --- /dev/null +++ b/src/tests/testdata/cs/iso-8859-2.txt @@ -0,0 +1,4 @@ +Ledek n (Alcedo atthis) je prmrn 16,5 cm velk ptk z eledi +ledkovitch (Alcedinidae). Je velmi vrazn zbarven s oranovou spodinou a +modrm hbetem, kdly a temenem. Vraznm znakem je tak jeho npadn dlouh +zapiatl zobk. Pro sv krsn zbarven je nazvn Ltajc drahokam. diff --git a/src/tests/testdata/cs/maccentraleurope.txt b/src/tests/testdata/cs/maccentraleurope.txt new file mode 100644 index 0000000..4af3ef8 --- /dev/null +++ b/src/tests/testdata/cs/maccentraleurope.txt @@ -0,0 +1,4 @@ +Ledˇek ޒn (Alcedo atthis) je prmrn 16,5 cm velk ptk z eledi +ledˇkovitch (Alcedinidae). Je velmi vrazn zbarven s oranovou spodinou a +modrm hbetem, kޒdly a temenem. Vraznm znakem je tak jeho npadn dlouh +zapiatl zobk. Pro sv krsn zbarven je nazvn Ltajc drahokam. diff --git a/src/tests/testdata/cs/utf-8.txt b/src/tests/testdata/cs/utf-8.txt new file mode 100644 index 0000000..9b61d5e --- /dev/null +++ b/src/tests/testdata/cs/utf-8.txt @@ -0,0 +1,4 @@ +Ledňáček říční (Alcedo atthis) je průměrně 16,5 cm velký pták z čeledi +ledňáčkovitých (Alcedinidae). Je velmi výrazně zbarvený s oranžovou spodinou a +modrým hřbetem, křídly a temenem. Výrazným znakem je také jeho nápadně dlouhý +zašpičatělý zobák. Pro své krásné zbarvení je nazýván Létající drahokam. diff --git a/src/tests/testdata/cs/windows-1250.txt b/src/tests/testdata/cs/windows-1250.txt new file mode 100644 index 0000000..43ac463 --- /dev/null +++ b/src/tests/testdata/cs/windows-1250.txt @@ -0,0 +1,4 @@ +Ledek n (Alcedo atthis) je prmrn 16,5 cm velk ptk z eledi +ledkovitch (Alcedinidae). Je velmi vrazn zbarven s oranovou spodinou a +modrm hbetem, kdly a temenem. Vraznm znakem je tak jeho npadn dlouh +zapiatl zobk. Pro sv krsn zbarven je nazvn Ltajc drahokam. diff --git a/src/tests/testdata/cz/ISO-8859-2/wikitop_cz_ISO-8859-2.txt b/src/tests/testdata/cz/ISO-8859-2/wikitop_cz_ISO-8859-2.txt deleted file mode 100644 index 4d4a787..0000000 --- a/src/tests/testdata/cz/ISO-8859-2/wikitop_cz_ISO-8859-2.txt +++ /dev/null @@ -1,417 +0,0 @@ -Hlavn strana - - Z Wikipedie, oteven encyklopedie - Skoit na: Navigace, Hledn - - Vtejte ve Wikipedii, - internetov encyklopedii, kterou me kad upravovat. - - Kultura Kultura - Geografie Geografie - Historie Historie - - Matematika Matematika - Lid Lid - Filosofie Filosofie - - Vda Vda - Spolenost Spolenost - Technologie Technologie - esk verze Wikipedie zahjila svoji innost v roce 2002, nyn m ji - 209 883 lnk. - Prohldnte si nejlep z nich, na pskoviti si vyzkouejte, jak se - edituje, nebo diskutujte na strnce Pod lpou. - Prvodce Npovda Ptte se (FAQ) AZ Portl Wikipedie (rychl - orientace) Oznmen Pod lpou Kontakt - Other languages icon.svg - Jin jazyky | Embassy - Clanektydne.gif - lnek tdne - HMS Victory v bitv u Trafalgaru - - England expects that every man will do his duty (esky Anglie - oekv, e kad mu spln svou povinnost) byl nmon vlajkov - signl vyslan z vlajkov lodi viceadmirla Horatia Nelsona HMS Victory - jako zahjen bitvy u Trafalgaru proti nmonictvu napoleonsk Francie - 21. jna 1805. Bitva u Trafalgaru byla osudov pro Napoleonovo - Francouzsk csastv jako nmon velmoc. Spojen krlovstv zskalo - pevahu na moi a zmailo tak vechny nadje na francouzsk vylodn na - britskch ostrovech. - - Jakmile se britsk lod piblily ke spojenmu neptelskmu lostvu, - viceadmirl Nelson signalizoval ze sv lodi nezbytn instrukce ostatnm - lodm britskho lostva. Vdom si nastvajcch nesnadnch udlost, - povaoval za nezbytn uinit nco zvltnho. Nadil proto svmu - signlnmu dstojnkovi, poruku Johnu Pascovi, aby co nejrychleji - signalizoval ostatnm zprvu England confides that every man will do - his duty Anglie spolh, e kad mu spln svou povinnost. - Poruk Pasco viceadmirlovi navrhl zamnit vraz confides za - expects, protoe signl expect se ji nachzel v knize nmonch - kd, zatmco slovo confides by se muselo signalizovat psmeno po - psmenu. Nelson se zmnou souhlasil, pestoe vraz expect nevyjadoval - takovou mru osobn zodpovdnosti, jakou by si bval pl. - Akoliv ohledn pesnho znn signlu se po bitv vedla ada spor, - vznam vtzstv u Trafalgaru a smrt viceadmirla Nelsona v bitv - zpsobily, e se toto slov mezi Brity rozilo a bv od t doby - asto uvno a parafrzovno a do dnench dn. - Archiv Nejlep lnky Dal dobr lnky - Viteze.gif - Vte, e - Mlad plodnice vclavky obecn - * podhoub vclavky obecn v noci svtlkuje? - * dirigent Jevgenij Mravinskij ml fbii z vystupovn a nkdy - penechval na posledn chvli zen koncert svm pomocnkm? - * prvn bojov let uskutenil 22. jna 1911 kapitn Riccardo Moizo? - * Karibsk Nizozemsko nepouv jako svoji mnu euro, nbr americk - dolar? - - Nejnovj lnky Dal zajmavosti - Kategorie.gif - Listovn kategoriemi - - Exquisite-opera.png - -Prodn vdy - - Astronomie Biologie Fyzika Geografie Geologie Chemie - Informatika Logika Matematika - Gohomenew.png - -Humanitn a spoleensk vdy - - Architektura Ekonomie Filosofie Historie Lingvistika - Pedagogika Prvo Psychologie Sociologie Teologie - Exquisite-kfm home.png - -Znalosti a technika - - Doprava Dorozumvn Obchod Prmysl Vojenstv Zdravotnictv - Zemdlstv - Exquisite-amorok.png - -Kultura a spolenost - - Divadlo Film Fotografie Gastronomie Hudba Lid Literatura - Mytologie Nboenstv Politika Sport Tanec Turistika - Umn Zliby - -Jin rejstky - - Seznamy Kalend Stalet a roky Vda Vdy o Zemi Strom - kategori Vechny kategorie Portly - Dal informace - Portal.svg - Tematick portly - * Geografie: Albnie zerbjdn Balkn Belgie Bosna a - Hercegovina Brno ern Hora esko na Estonsko EU - Faersk ostrovy Finsko Francie Gruzie Hradec Krlov - Chorvatsko Indie rn Irsko Island Itlie Izrael - Japonsko Kanada Karlovy Vary Kazachstn Korea Kosovo - Krom Latinsk Amerika Liberec Litva Lotysko - Lucembursko Maarsko Makedonie Morava Most Nmecko - Nizozemsko Omn Ostrava Plze Polsko Portugalsko - Praha Prostjov Rakousko Rusko ecko Skotsko - Slovensko Slovinsko Spojen krlovstv Srbsko Stedn - Amerika a Karibik luknovsk vbek panlsko vdsko - vcarsko Tibet Teb Tebsko USA Vatikn Vietnam - Vykov Washington - - * Vdy: Astronomie Biologie Ekonomie Fytoenergetika a - kompostrenstv Fyzika Chemie Informan vda a - knihovnictv Jazyk Kosmonautika Matematika Medicna - Planetrn vdy Prvo Psychologie - - * Historie: Pravk Starovk Starovk Egypt Starovk m - Antika Stedovk Byzantsk e Kov vpravy Osmansk - e Novovk Kolonialismus Prvn svtov vlka - eskoslovensk opevnn Druh svtov vlka Komunistick reim - v eskoslovensku - - * Umn: Anime & Manga Nbytkstv Harry Potter Hudba - Hvzdn brna Film Fotografie Literatura Simpsonovi Star - Trek Televize Tolkien Vetelec vs. Predtor - - * Nboenstv: Bible Hebraistika Kesanstv Buddhismus - Hinduismus Jga - - * Doprava: Autobusov Automobil Letectv Lostvo Metro - Tramvajov Trolejbusov eleznice - - * Sport: Florbal Formule 1 Fotbal Ledn hokej Orientan - bh Ralye achy Tenis - - * Proda: Dinosaui Entomologie Houby Kon Psi Ptci - Rostliny ivoichov ivotn prosted - - * Ostatn: BDSM aj esk jazyk Filosofie Linux Lid - Pivo Potaov hry Politika Sexualita Svobodn software - Tba Vlka Zahrada a zahradnictv - - Dal informace - Crystal 128 xpaint.gif - Obrzek tdne - Ledovec Gornergletscher - Ledovec Gornergletscher nedaleko Zermattu ve vcarsku - Archiv Dal informace - Aktuality.gif - Aktuality - - 21. jen ptek - - * Flag of Slovenia.svg Slovinsk prezident Danilo Trk minutu po - plnoci rozpustil Sttn shromdn Republiky Slovinsko a vyhlsil - pedasn volby. - - 20. jen tvrtek - - * Flag of Libya.svg Stelnm zrannm pi zatkn podlehl libyjsk - vdce plukovnk Muammar Kaddf. - - 18. jna ter - - * Flag of Israel.svg Flag of Hamas.svg Po vce ne pti letech - vznn Hamsem byl proputn izraelsk vojk Gilad alit. - Podmnkou vmny je proputn vce ne 1000 palestinskch vzn a - tyto nerovn podmnky jsou pinou siln kritiky ze strany - izraelskch radikl. - - 15. jna sobota - - * Flag of World.svg Po celm svt vypukly v rmci projektu Jednotn - za zmnu svta, inspirovanho hnutm Occupy Wall Street, - demonstrace, na kterch lid protestuj proti vldnm krtm a - dajn nemravnmu chovn velkch finannch korporac. Demonstrace - byly naplnovny v 951 mstech, mezi nimi nechybla ani Praha. - * Flag of Europe.svg Flag of Greece.svg Nvrh na zdvojnsoben jmn - Mezinrodnho mnovho fondu tak, aby mohl inn pomoci - zadluenmu ecku, byl na zasedn ministr financ zem G20 - zamtnut. lo o nvrh Brazlie, Ruska, Indie, ny a Jihoafrick - republiky. Neproel hlavn kvli odporu Spojench stt a Kanady. - - 13. jna tvrtek - - * Flag of Slovakia.svg Slovensk parlament schvlil pspvek do - zchrannho fondu eurozny (EFSF) na pomoc zadluenm zemm ve vi - 7,7 miliardy eur (190 miliard K). Poslanci ji pedtm odhlasovali - vldn nvrh stavnho zkona o zkrcen volebnho obdob, kter - umon konn pedasnch voleb. Opozin sociln demokrat za to - vld slbili podporu schvlen eurofondu. - - 11. jna ter - - * Flag of Slovakia.svg Slovensk vlda Ivety Radiov prohrla - hlasovn o dve, se kterm spojila schvalovn pomoci - zadluenmu ecku. - * Flag of Ukraine.svg Bval premirka Ukrajiny Julija Tymoenkov - byla odsouzena k sedmiletmu trestu vzen za pekroen pravomoc - pi podpisu smlouvy o dodvkch ruskho plynu. Podrobnj lnek - na Wikizprvch: Tymoenkov m jt na sedm let do vzen - - 10. jna pondl - - * Flag of the Czech Republic.svg Vtzem 17. ronku soute Czech - Press Photo se stal Stanislav Krupa z asopisu Reflex se snmkem z - nepokoj na luknovsku. - * Flag of Sweden.svg Flag of the United States.svg Laurety Nobelovy - cena za ekonomii se stali amerit ekonomov Christopher A. Sims a - Thomas J. Sargent. Podrobnj lnek na Wikizprvch: Sargent a - Sims vyznamenni Nobelovou cenou za ekonomii - - Wikizprvy Wikizprvy - Exquisite-kalarm.gif - 21. jen v minulosti - - HoratioNelson1.jpg - * 1125 Zemel Kosmas, autor Kroniky esk (Chronica Boemorum). - * 1435 Jan Rokycana byl eskm snmem zvolen za spolenho - praskho arcibiskupa katolk i kalink. - * 1600 Iejasu Tokugawa porazil vdce soupecch japonskch klan - v bitv u Sekigahary, co je pokldno za potek guntu - Tokugawa. - * 1805 Bitva u Trafalgaru: admirl Nelson (na obrzku) rozdrtil - v ele britskho lostva francouzsk a panlsk nmon sly. - * 1940 Uskutenila se prvn oficiln jzda visut lanovky - Tatransk Lomnica Lomnick tt. - * 1941 eskoslovensk p prapor 11 Vchodn pod velenm - podplukovnka Karla Klaplka poslil obranu Tobrku. - - Dal vro - Redakce.gif - Redakce - - Pomoc - Vtejte ve Wikipedii Rady a informace pro pispvatele asto - kladen otzky Jak vytvoit lnek Jak editovat strnku Vzhled a - styl Obrzky Pravopisn pravidla O kategorich Psan dat - ast chyby Potebuji pomoc Wikipedist Diskuse o smazn - ablony Jak citovat Wikipedii - - O Wikipedii - Poadovan lnky Uiten informace Kompletn seznam lnk - Velvyslanectv Wikipedie - Dal informace - Nuvola apps kdf.gif - Ostatn projekty - - Wikipedii provozuje neziskov nadace Wikimedia Foundation, kter - spravuje i adu dalch otevench mnohojazynch wiki projekt: - logo Wikislovnk logo Wikizdroje logo Wikicitty logo Wikiknihy logo - Wikizprvy logo Wikiverzita logo Wikidruhy logo Wikimedia Commons logo - Meta-Wiki - Wikislovnk Wikizdroje Wikicitty Wikiknihy Wikizprvy Wikiverzita - Wikidruhy Commons Meta-Wiki - Vkladov slovnk Voln dostupn dokumenty Sbrka citt Voln dostupn - knihy a manuly Oteven zpravodajstv Voln dostupn vzdln Adres - biologickch druh Sdlen sbrka mdi Koordinace vech projekt - Wikimedia - Crystal 128 locale.gif - Ostatn jazyky - - Flag of the United Kingdom.svg Anglitina (English) Flag of - France.svg Francouztina (Franais) Flag of Germany.svg Nmina - (Deutsch) Flag of Poland.svg Poltina (Polski) Flag of Russia.svg -Rutina () Flag of Slovakia.svg Sloventina (Slovenina) - Flag of Spain.svg panltina (Espaol) - - Vybran dal jazyky (Wikipedie s vce ne 150 000 lnky jsou uvedeny - tun): - - Anglitina zjednoduen (Simple English) Arabtina - ( Al-'arabiy) Baskitina (Euskara) Blorutina - (/Belaruskaja) Bulhartina (/Blgarski) ntina - ( Zhngwn) Dntina (Dansk) Dolnoluick srbtina (Dolnoserbski) - Esperanto Estontina (Eesti) Fintina (Suomi) Galicijtina - (Galego) Hebrejtina ( 'Ivrt) Hindtina (Hind) - Hornoluick srbtina (Hornjoserbsce) Chorvattina (Hrvatski) Ido - Indontina (Bahasa Indonesia) Interlingua Irtina (Gaeilge) - Italtina (Italiano) Japontina ( Nihongo) Katalntina (Catal) - Korejtina ( Hangukeo) Latina Litevtina (Lietuvi) - Lotytina (Latvieu) Maartina (Magyar) Malajtina (Bahasa Melayu) - Nizozemtina (Nederlands) Nortina (Norsk nynorsk) - Nortina (Norsk bokml) Pertina ( Prsi) Portugaltina - (Portugus) Rumuntina (Romn) Rusntina () - etina (/Ellinika) Sleztina (lnski) - Slovintina (Slovenina) Srbtina (/srpski) vdtina - (Svenska) Thajtina ( phasa thai) Turetina (Trke) - Ukrajintina (/Ukrajinska) Veltina (Cymraeg) - Vietnamtina (Ting Vit) - __________________________________________________________________ - - Seznam jazyk Wikipedie - - Vechny lnky | Nov lnky | Sirot strnky | Kategorie | Obnovit - cache - -NAHORU - - Citovno z - http://cs.wikipedia.org/w/index.php?title=Hlavn%C3%AD_strana&oldid=630 - 0268 - -Osobn nstroje - - * Pihlen / vytvoen tu - -Jmenn prostory - - * lnek - * Diskuse - -Varianty - -Zobrazen - - * st - * Zobrazit zdroj - * Zobrazit historii - -Akce - -Hledn - - ____________________ (Submit) Hledat - -Navigace - - * Hlavn strana - * Portl Wikipedie - * Aktuality - * Pod lpou - * Posledn zmny - * Nhodn lnek - * Npovda - * Podpote Wikipedii - -Tisk/export - - * Vytvoit knihu - * Sthnout jako PDF - * Verze k tisku - -Nstroje - - * Odkazuje sem - * Souvisejc zmny - * Nast soubor - * Speciln strnky - * Trval odkaz - * Citovat strnku - -V jinch jazycch - - * - * - * Catal - * Dansk - * Deutsch - * Dolnoserbski - * - * English - * Espaol - * Eesti - * Esperanto - * Euskara - * - * Suomi - * Franais - * Galego - * - * Hrvatski - * Hornjoserbsce - * Magyar - * Bahasa Indonesia - * Italiano - * - * - * Lietuvi - * Bahasa Melayu - * Nederlands - * Norsk (nynorsk) - * Norsk (bokml) - * Polski - * Portugus - * Romn - * - * Slovenina - * Slovenina - * Simple English - * / Srpski - * Svenska - * Trke - * - * Ting Vit - * - * - - * Strnka byla naposledy editovna 5. 1. 2011 v 12:51. - * Text je dostupn pod licenc Creative Commons Uvete autora - Zachovejte licenci 3.0 Unported, ppadn za dalch podmnek. - Podrobnosti naleznete na strnce Podmnky uit. - - * Ochrana osobnch daj - * O Wikipedii - * Vylouen odpovdnosti - * Mobiln verze - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/cz/UTF-8/wikitop_cz_UTF-8.txt b/src/tests/testdata/cz/UTF-8/wikitop_cz_UTF-8.txt deleted file mode 100644 index ec4b8c4..0000000 --- a/src/tests/testdata/cz/UTF-8/wikitop_cz_UTF-8.txt +++ /dev/null @@ -1,415 +0,0 @@ - Z Wikipedie, otevřené encyklopedie - Skočit na: Navigace, Hledání - - Vítejte ve Wikipedii, - internetové encyklopedii, kterou může každý upravovat. - - Kultura Kultura - Geografie Geografie - Historie Historie - - Matematika Matematika - Lidé Lidé - Filosofie Filosofie - - Věda Věda - Společnost Společnost - Technologie Technologie - Česká verze Wikipedie zahájila svoji činnost v roce 2002, nyní má již - 209 883 článků. - Prohlédněte si nejlepší z nich, na pískovišti si vyzkoušejte, jak se - edituje, nebo diskutujte na stránce Pod lípou. - Průvodce • Nápověda • Ptáte se (FAQ) • A–Z • Portál Wikipedie (rychlá - orientace) • Oznámení • Pod lípou • Kontakt - Other languages icon.svg - Jiné jazyky | Embassy - Clanektydne.gif - Článek týdne - HMS Victory v bitvě u Trafalgaru - - „England expects that every man will do his duty“ (česky „Anglie - očekává, že každý muž splní svou povinnost“) byl námořní vlajkový - signál vyslaný z vlajkové lodi viceadmirála Horatia Nelsona HMS Victory - jako zahájení bitvy u Trafalgaru proti námořnictvu napoleonské Francie - 21. října 1805. Bitva u Trafalgaru byla osudová pro Napoleonovo - Francouzské císařství jako námořní velmoc. Spojené království získalo - převahu na moři a zmařilo tak všechny naděje na francouzské vylodění na - britských ostrovech. - - Jakmile se britské lodě přiblížily ke spojenému nepřátelskému loďstvu, - viceadmirál Nelson signalizoval ze své lodi nezbytné instrukce ostatním - lodím britského loďstva. Vědom si nastávajících nesnadných událostí, - považoval za nezbytné učinit něco zvláštního. Nařídil proto svému - signálnímu důstojníkovi, poručíku Johnu Pascovi, aby co nejrychleji - signalizoval ostatním zprávu „England confides that every man will do - his duty“ – „Anglie spoléhá, že každý muž splní svou povinnost“. - Poručík Pasco viceadmirálovi navrhl zaměnit výraz „confides“ za - „expects“, protože signál „expect“ se již nacházel v knize námořních - kódů, zatímco slovo „confides“ by se muselo signalizovat písmeno po - písmenu. Nelson se změnou souhlasil, přestože výraz expect nevyjadřoval - takovou míru osobní zodpovědnosti, jakou by si býval přál. - Ačkoliv ohledně přesného znění signálu se po bitvě vedla řada sporů, - význam vítězství u Trafalgaru a smrt viceadmirála Nelsona v bitvě - způsobily, že se toto úsloví mezi Brity rozšířilo a bývá od té doby - často užíváno a parafrázováno až do dnešních dní. - Archiv • Nejlepší články • Další dobré články… - Viteze.gif - Víte, že… - Mladé plodnice václavky obecné - * …podhoubí václavky obecné v noci světélkuje? - * …dirigent Jevgenij Mravinskij měl fóbii z vystupování a někdy - přenechával na poslední chvíli řízení koncertů svým pomocníkům? - * …první bojový let uskutečnil 22. října 1911 kapitán Riccardo Moizo? - * …Karibské Nizozemsko nepoužívá jako svoji měnu euro, nýbrž americký - dolar? - - Nejnovější články • Další zajímavosti… - Kategorie.gif - Listování kategoriemi - - Exquisite-opera.png - -Přírodní vědy - - Astronomie • Biologie • Fyzika • Geografie • Geologie • Chemie • - Informatika • Logika • Matematika - Gohomenew.png - -Humanitní a společenské vědy - - Architektura • Ekonomie • Filosofie • Historie • Lingvistika • - Pedagogika • Právo • Psychologie • Sociologie • Teologie - Exquisite-kfm home.png - -Znalosti a technika - - Doprava • Dorozumívání • Obchod • Průmysl • Vojenství • Zdravotnictví • - Zemědělství - Exquisite-amorok.png - -Kultura a společnost - - Divadlo • Film • Fotografie • Gastronomie • Hudba • Lidé • Literatura • - Mytologie • Náboženství • Politika • Sport • Tanec • Turistika • - Umění • Záliby - -Jiné rejstříky - - Seznamy • Kalendář • Staletí a roky • Věda • Vědy o Zemi • Strom - kategorií • Všechny kategorie • Portály - Další informace… - Portal.svg - Tematické portály - * Geografie: Albánie • Ázerbájdžán • Balkán • Belgie • Bosna a - Hercegovina • Brno • Černá Hora • Česko • Čína • Estonsko • EU • - Faerské ostrovy • Finsko • Francie • Gruzie • Hradec Králové • - Chorvatsko • Indie • Írán • Irsko • Island • Itálie • Izrael • - Japonsko • Kanada • Karlovy Vary • Kazachstán • Korea • Kosovo • - Kroměříž • Latinská Amerika • Liberec • Litva • Lotyšsko • - Lucembursko • Maďarsko • Makedonie • Morava • Most • Německo • - Nizozemsko • Omán • Ostrava • Plzeň • Polsko • Portugalsko • - Praha • Prostějov • Rakousko • Rusko • Řecko • Skotsko • - Slovensko • Slovinsko • Spojené království • Srbsko • Střední - Amerika a Karibik • Šluknovský výběžek • Španělsko • Švédsko • - Švýcarsko • Tibet • Třebíč • Třebíčsko • USA • Vatikán • Vietnam • - Vyškov • Washington - - * Vědy: Astronomie • Biologie • Ekonomie • Fytoenergetika a - kompostárenství • Fyzika • Chemie • Informační věda a - knihovnictví • Jazyk • Kosmonautika • Matematika • Medicína • - Planetární vědy • Právo • Psychologie - - * Historie: Pravěk • Starověk • Starověký Egypt • Starověký Řím • - Antika • Středověk • Byzantská říše • Křížové výpravy • Osmanská - říše • Novověk • Kolonialismus • První světová válka • - Československé opevnění • Druhá světová válka • Komunistický režim - v Československu - - * Umění: Anime & Manga • Nábytkářství • Harry Potter • Hudba • - Hvězdná brána • Film • Fotografie • Literatura • Simpsonovi • Star - Trek • Televize • Tolkien • Vetřelec vs. Predátor - - * Náboženství: Bible • Hebraistika • Křesťanství • Buddhismus • - Hinduismus • Jóga - - * Doprava: Autobusová • Automobil • Letectví • Loďstvo • Metro • - Tramvajová • Trolejbusová • Železnice - - * Sport: Florbal • Formule 1 • Fotbal • Lední hokej • Orientační - běh • Ralye • Šachy • Tenis - - * Příroda: Dinosauři • Entomologie • Houby • Koně • Psi • Ptáci • - Rostliny • Živočichové • Životní prostředí - - * Ostatní: BDSM • Čaj • Český jazyk • Filosofie • Linux • Lidé • - Pivo • Počítačové hry • Politika • Sexualita • Svobodný software • - Těžba • Válka • Zahrada a zahradnictví - - Další informace… - Crystal 128 xpaint.gif - Obrázek týdne - Ledovec Gornergletscher - Ledovec Gornergletscher nedaleko Zermattu ve Švýcarsku - Archiv • Další informace… - Aktuality.gif - Aktuality - - 21. říjen – pátek - - * Flag of Slovenia.svg Slovinský prezident Danilo Türk minutu po - půlnoci rozpustil Státní shromáždění Republiky Slovinsko a vyhlásil - předčasné volby. - - 20. říjen – čtvrtek - - * Flag of Libya.svg Střelným zraněním při zatýkání podlehl libyjský - vůdce plukovník Muammar Kaddáfí. - - 18. října – úterý - - * Flag of Israel.svg Flag of Hamas.svg Po více než pěti letech - věznění Hamásem byl propuštěn izraelský voják Gilad Šalit. - Podmínkou výměny je propuštění více než 1000 palestinských věznů a - tyto nerovné podmínky jsou příčinou silné kritiky ze strany - izraelských radikálů. - - 15. října – sobota - - * Flag of World.svg Po celém světě vypukly v rámci projektu „Jednotně - za změnu světa“, inspirovaného hnutím Occupy Wall Street, - demonstrace, na kterých lidé protestují proti vládním škrtům a - údajně nemravnému chování velkých finančních korporací. Demonstrace - byly naplánovány v 951 městech, mezi nimiž nechyběla ani Praha. - * Flag of Europe.svg Flag of Greece.svg Návrh na zdvojnásobení jmění - Mezinárodního měnového fondu tak, aby mohl účinně pomoci - zadluženému Řecku, byl na zasedání ministrů financí zemí G20 - zamítnut. Šlo o návrh Brazílie, Ruska, Indie, Číny a Jihoafrické - republiky. Neprošel hlavně kvůli odporu Spojených států a Kanady. - - 13. října – čtvrtek - - * Flag of Slovakia.svg Slovenský parlament schválil příspěvek do - záchranného fondu eurozóny (EFSF) na pomoc zadluženým zemím ve výši - 7,7 miliardy eur (190 miliard Kč). Poslanci již předtím odhlasovali - vládní návrh ústavního zákona o zkrácení volebního období, který - umožní konání předčasných voleb. Opoziční sociální demokraté za to - vládě slíbili podporu schválení eurofondu. - - 11. října – úterý - - * Flag of Slovakia.svg Slovenská vláda Ivety Radičové prohrála - hlasování o důvěře, se kterým spojila schvalování pomoci - zadluženému Řecku. - * Flag of Ukraine.svg Bývalá premiérka Ukrajiny Julija Tymošenková - byla odsouzena k sedmiletému trestu vězení za překročení pravomocí - při podpisu smlouvy o dodávkách ruského plynu. Podrobnější článek - na Wikizprávách: Tymošenková má jít na sedm let do vězení - - 10. října – pondělí - - * Flag of the Czech Republic.svg Vítězem 17. ročníku soutěže Czech - Press Photo se stal Stanislav Krupař z časopisu Reflex se snímkem z - nepokojů na Šluknovsku. - * Flag of Sweden.svg Flag of the United States.svg Laureáty Nobelovy - cena za ekonomii se stali američtí ekonomové Christopher A. Sims a - Thomas J. Sargent. Podrobnější článek na Wikizprávách: Sargent a - Sims vyznamenáni Nobelovou cenou za ekonomii - - Wikizprávy Wikizprávy - Exquisite-kalarm.gif - 21. říjen v minulosti - - HoratioNelson1.jpg - * 1125 – Zemřel Kosmas, autor Kroniky české (Chronica Boemorum). - * 1435 – Jan Rokycana byl českým sněmem zvolen za společného - pražského arcibiskupa katolíků i kališníků. - * 1600 – Iejasu Tokugawa porazil vůdce soupeřících japonských klanů - v bitvě u Sekigahary, což je pokládáno za počátek šógunátu - Tokugawa. - * 1805 – Bitva u Trafalgaru: admirál Nelson (na obrázku) rozdrtil - v čele britského loďstva francouzské a španělské námořní síly. - * 1940 – Uskutečnila se první oficiální jízda visuté lanovky - Tatranská Lomnica – Lomnický štít. - * 1941 – Československý pěší prapor 11 – Východní pod velením - podplukovníka Karla Klapálka posílil obranu Tobrúku. - - Další výročí… - Redakce.gif - Redakce - - Pomoc - Vítejte ve Wikipedii • Rady a informace pro přispěvatele • Často - kladené otázky • Jak vytvořit článek • Jak editovat stránku • Vzhled a - styl • Obrázky • Pravopisná pravidla • O kategoriích • Psaní dat • - Časté chyby • Potřebuji pomoc • Wikipedisté • Diskuse o smazání • - Šablony • Jak citovat Wikipedii - - O Wikipedii - Požadované články • Užitečné informace • Kompletní seznam článků • - Velvyslanectví Wikipedie - Další informace… - Nuvola apps kdf.gif - Ostatní projekty - - Wikipedii provozuje nezisková nadace Wikimedia Foundation, která - spravuje i řadu dalších otevřených mnohojazyčných wiki projektů: - logo Wikislovník logo Wikizdroje logo Wikicitáty logo Wikiknihy logo - Wikizprávy logo Wikiverzita logo Wikidruhy logo Wikimedia Commons logo - Meta-Wiki - Wikislovník Wikizdroje Wikicitáty Wikiknihy Wikizprávy Wikiverzita - Wikidruhy Commons Meta-Wiki - Výkladový slovník Volně dostupné dokumenty Sbírka citátů Volně dostupné - knihy a manuály Otevřené zpravodajství Volně dostupné vzdělání Adresář - biologických druhů Sdílená sbírka médií Koordinace všech projektů - Wikimedia - Crystal 128 locale.gif - Ostatní jazyky - - Flag of the United Kingdom.svg Angličtina (English) · Flag of - France.svg Francouzština (Français) · Flag of Germany.svg Němčina - (Deutsch) · Flag of Poland.svg Polština (Polski) · Flag of Russia.svg -Ruština (Русский) · Flag of Slovakia.svg Slovenština (Slovenčina) · - Flag of Spain.svg Španělština (Español) - - Vybrané další jazyky (Wikipedie s více než 150 000 články jsou uvedeny - tučně): - - Angličtina zjednodušená (Simple English) · Arabština - (العربيّة Al-'arabiyâ) · Baskičtina (Euskara) · Běloruština - (Беларуская/Belaruskaja) · Bulharština (Български/Bălgarski) · Čínština - (中文 Zhōngwén) · Dánština (Dansk) · Dolnolužická srbština (Dolnoserbski) - · Esperanto · Estonština (Eesti) · Finština (Suomi) · Galicijština - (Galego) · Hebrejština (עברית 'Ivrīt) · Hindština (Hindī) · - Hornolužická srbština (Hornjoserbsce) · Chorvatština (Hrvatski) · Ido · - Indonéština (Bahasa Indonesia) · Interlingua · Irština (Gaeilge) · - Italština (Italiano) · Japonština (日本語 Nihongo) · Katalánština (Català) - · Korejština (한국어 Hangukeo) · Latina · Litevština (Lietuvių) · - Lotyština (Latviešu) · Maďarština (Magyar) · Malajština (Bahasa Melayu) - · Nizozemština (Nederlands) · Norština (Norsk nynorsk) · - Norština (Norsk bokmål) · Perština (فارسی Pârsi) · Portugalština - (Português) · Rumunština (Română) · Rusínština (Русиньскый) · - Řečtina (Ελληνικά/Ellinika) · Slezština (Ślůnski) · - Slovinština (Slovenščina) · Srbština (српски/srpski) · Švédština - (Svenska) · Thajština (ไทย phasa thai) · Turečtina (Türkçe) · - Ukrajinština (Українська/Ukrajinska) · Velština (Cymraeg) · - Vietnamština (Tiếng Việt) - __________________________________________________________________ - - Seznam jazyků Wikipedie - - Všechny články | Nové články | Sirotčí stránky | Kategorie | Obnovit - cache - -NAHORU - - Citováno z - „http://cs.wikipedia.org/w/index.php?title=Hlavn%C3%AD_strana&oldid=630 - 0268“ - -Osobní nástroje - - * Přihlášení / vytvoření účtu - -Jmenné prostory - - * Článek - * Diskuse - -Varianty - -Zobrazení - - * Číst - * Zobrazit zdroj - * Zobrazit historii - -Akce - -Hledání - - ____________________ (Submit) Hledat - -Navigace - - * Hlavní strana - * Portál Wikipedie - * Aktuality - * Pod lípou - * Poslední změny - * Náhodný článek - * Nápověda - * Podpořte Wikipedii - -Tisk/export - - * Vytvořit knihu - * Stáhnout jako PDF - * Verze k tisku - -Nástroje - - * Odkazuje sem - * Související změny - * Načíst soubor - * Speciální stránky - * Trvalý odkaz - * Citovat stránku - -V jiných jazycích - - * العربية - * Български - * Català - * Dansk - * Deutsch - * Dolnoserbski - * Ελληνικά - * English - * Español - * Eesti - * Esperanto - * Euskara - * فارسی - * Suomi - * Français - * Galego - * עברית - * Hrvatski - * Hornjoserbsce - * Magyar - * Bahasa Indonesia - * Italiano - * 日本語 - * 한국어 - * Lietuvių - * Bahasa Melayu - * Nederlands - * ‪Norsk (nynorsk)‬ - * ‪Norsk (bokmål)‬ - * Polski - * Português - * Română - * Русский - * Slovenčina - * Slovenščina - * Simple English - * Српски / Srpski - * Svenska - * Türkçe - * ไทย - * Tiếng Việt - * Українська - * 中文 - - * Stránka byla naposledy editována 5. 1. 2011 v 12:51. - * Text je dostupný pod licencí Creative Commons Uveďte autora – - Zachovejte licenci 3.0 Unported, případně za dalších podmínek. - Podrobnosti naleznete na stránce Podmínky užití. - - * Ochrana osobních údajů - * O Wikipedii - * Vyloučení odpovědnosti - * Mobilní verze - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/da/iso-8859-1.txt b/src/tests/testdata/da/iso-8859-1.txt new file mode 100644 index 0000000..f36d4ab --- /dev/null +++ b/src/tests/testdata/da/iso-8859-1.txt @@ -0,0 +1,7 @@ +Dansk er et nord-germansk sprog af den stnordiske (kontinentale) gruppe, der +tales af ca. seks millioner mennesker. Det er strkt pvirket af plattysk. Dansk +tales ogs i Sydslesvig (i Flensborg ca. 20 %) samt p Frerne og Grnland [1]. +Dansk er tt forbundet med norsk. Fra et sprogvidenskabeligt synspunkt kan den +fremherskende form af norsk, bokml (og i endnu hjere grad riksml), betragtes +som dansk, i hvert fald hvad skriftsproget angr. Bde dansk, norsk og svensk er +skandinaviske sprog og minder meget om hinanden. diff --git a/src/tests/testdata/da/iso-8859-15.txt b/src/tests/testdata/da/iso-8859-15.txt new file mode 100644 index 0000000..c400e0a --- /dev/null +++ b/src/tests/testdata/da/iso-8859-15.txt @@ -0,0 +1,10 @@ +Eurosymbolet eller eurotegnet () anvendes som valutasymbol for mntenheden +euro. Symbolsk kombinerer det et E eller et grsk epsilon med de to parallelle +streger, man ofte ser i valutasymboler. + +Det vides ikke med sikkerhed, hvem eurosymbolet blev designet af. Nogle medier +hvder, det blev skabt af tidligere designer ved EF Arthur Eisenmenger, mens +andre pstr, det blev skabt af en lille gruppe ledet af Alain Billiet. Muligvis +er ingen af disse forklaringer korrekte, da Den Paneuropiske Union udsendte en +'1 euro'-medalje i 1972, hvorp man kan se et symbol, der i hj grad ligner det +nuvrende eurosymbol. diff --git a/src/tests/testdata/da/utf-8.txt b/src/tests/testdata/da/utf-8.txt new file mode 100644 index 0000000..e5e0274 --- /dev/null +++ b/src/tests/testdata/da/utf-8.txt @@ -0,0 +1,10 @@ +Eurosymbolet eller eurotegnet (€) anvendes som valutasymbol for møntenheden +euro. Symbolsk kombinerer det et E eller et græsk epsilon med de to parallelle +streger, man ofte ser i valutasymboler. + +Det vides ikke med sikkerhed, hvem eurosymbolet blev designet af. Nogle medier +hævder, det blev skabt af tidligere designer ved EF Arthur Eisenmenger, mens +andre påstår, det blev skabt af en lille gruppe ledet af Alain Billiet. Muligvis +er ingen af disse forklaringer korrekte, da Den Paneuropæiske Union udsendte en +'1 euro'-medalje i 1972, hvorpå man kan se et symbol, der i høj grad ligner det +nuværende eurosymbol. diff --git a/src/tests/testdata/da/windows-1252.txt b/src/tests/testdata/da/windows-1252.txt new file mode 100644 index 0000000..db8faf1 --- /dev/null +++ b/src/tests/testdata/da/windows-1252.txt @@ -0,0 +1,10 @@ +Eurosymbolet eller eurotegnet () anvendes som valutasymbol for mntenheden +euro. Symbolsk kombinerer det et E eller et grsk epsilon med de to parallelle +streger, man ofte ser i valutasymboler. + +Det vides ikke med sikkerhed, hvem eurosymbolet blev designet af. Nogle medier +hvder, det blev skabt af tidligere designer ved EF Arthur Eisenmenger, mens +andre pstr, det blev skabt af en lille gruppe ledet af Alain Billiet. Muligvis +er ingen af disse forklaringer korrekte, da Den Paneuropiske Union udsendte en +'1 euro'-medalje i 1972, hvorp man kan se et symbol, der i hj grad ligner det +nuvrende eurosymbol. diff --git a/src/tests/testdata/de/UTF-8/wikitop_de_UTF-8.txt b/src/tests/testdata/de/UTF-8/wikitop_de_UTF-8.txt deleted file mode 100644 index ee1f197..0000000 --- a/src/tests/testdata/de/UTF-8/wikitop_de_UTF-8.txt +++ /dev/null @@ -1,219 +0,0 @@ - Wikipedia ist ein Projekt zum Aufbau einer Enzyklopädie aus freien - Inhalten in allen Sprachen der Welt. Jeder kann mit seinem Wissen - beitragen. Seit Mai 2001 sind so 1.301.685 Artikel in deutscher Sprache - entstanden. Gute Autorinnen und Autoren sind stets willkommen. - - Geographie Geschichte Gesellschaft Kunst und Kultur - Religion Sport Technik Wissenschaft - - Artikel nach Themen · Alphabetischer Index · Artikel nach Kategorien · - Gesprochene Wikipedia - - Kontakt · Presse · Statistik · Andere Sprachen · Mentorenprogramm - - Wikipedia aktuell - - Bis zum 31. Oktober können Stimmen für den Publikumspreis des - Schreibwettbewerbs abgegeben werden. - -Artikel des Tages - - Ku-ring-gai-Chase-Nationalpark - - Der Ku-ring-gai-Chase-Nationalpark ist ein Nationalpark bei Sydney in - Australien. Er wurde 1894 gegründet und ist damit einer der ältesten - Nationalparks Australiens. Der Park umfasst etwa 150 km² von Eukalyptus - dominiertes Buschland auf Sandstein-Höhenzügen sowie einige Rias, vom - Pazifischen Ozean überflutete Flusstäler. Traditionelle Eigentümer des - Landes waren die Guringai, ein Stamm der Aborigines. Im Nationalpark - befinden sich viele Artefakte ihrer Besiedlung, unter anderem - Petroglyphen im Stil der Sydney-Felsgravuren. Ab 1834 siedelten Weiße - im Gebiet; heute befindet sich mit Cottage Point aber nur noch ein - bewohnter Ort im Park. Für die Bewohner Sydneys ist der Nationalpark - ein beliebtes Naherholungsgebiet mit jährlich rund zwei Millionen - Besuchern. mehr - Archiv · Weitere exzellente und lesenswerte Artikel · RSS-Feed - RSS-Feed - -Was geschah am 21. Oktober? - - * Erzherzog Karl und Prinzessin Zita im Juli 1911 - 1911 – In Schwarzau am Steinfeld heiraten Erzherzog Karl von - Österreich und Prinzessin Zita von Bourbon-Parma, von 1916 bis 1918 - Österreichs letztes Kaiserpaar. - * 1941 – Massenerschießung von Kragujevac: Einheiten der deutschen - Wehrmacht ermorden mehr als 2300 serbische Zivilisten. - * 1971 – Dem chilenischen Dichter Pablo Neruda wird der Nobelpreis - für Literatur zuerkannt. - * 1986 – Die Afrikanische Charta der Menschenrechte und der Rechte - der Völker tritt in Kraft. - * 2006 – In Danzig nimmt sich eine am Tag zuvor vor ihrer Klasse von - Mitschülern sexuell erniedrigte 14-jährige Gymnasiastin das Leben. - - Weitere Ereignisse - -In den Nachrichten - - ETA · Occupy Wall Street · Türkisch-kurdischer Konflikt - __________________________________________________________________ - - Muammar al-Gaddafi (2009) - * Mit dem ersten Start einer russischen Sojus-Rakete vom europäischen - Weltraumbahnhof Kourou sind die ersten beiden Satelliten des - Navigationssatellitensystems Galileo ins All gebracht worden. - * Muammar al-Gaddafi (Bild), der frühere Machthaber Libyens, ist in - seiner Heimatstadt Sirte erschossen worden. - * Der britische Autor Julian Barnes hat für seinen Roman The Sense of - an Ending den renommierten Booker Prize erhalten. - - Weitere aktuelle Ereignisse · Wikinews - -Kürzlich Verstorbene - - * Jean Jülich (82), deutscher Widerstandskämpfer († 19. Oktober) - * Bohdan Osadczuk (91), ukrainischer Autor († 19. Oktober) - * Norman Corwin (101), US-amerikanischer Autor († 18. Oktober) - - Weitere kürzlich Verstorbene - -Schon gewusst? - - Westseite der Burg Friedestrom - * Die niederrheinische Burg Friedestrom in Zons ist sowohl Bau- als - auch Bodendenkmal. - * Über die Herkunft des Komponisten Sigismondo d’Indias ist wenig - bekannt, aber er gilt als Wegbereiter der Barockmusik in Italien. - * Die USS Newport News (CA-148) war der letzte Kreuzer im aktiven - Dienst der United States Navy, der nur mit Geschützen bewaffnet - war. - * Rosa Manus war über 30 Jahre lang bis zum deutschen Einmarsch 1940 - eine der wichtigsten Leitfiguren der niederländischen - Frauenbewegung. - - Weitere neue Artikel · Frühere Schon-gewusst-Artikel - -Schwesterprojekte - - Meta-Wiki – Koordination Commons – Medien - Wiktionary – Wörterbuch Wikibooks – Lehrbücher - Wikiquote – Zitate Wikiversity – Lernplattform - Wikisource – Quellen Wikinews – Nachrichten - - Von - „http://de.wikipedia.org/w/index.php?title=Wikipedia:Hauptseite&oldid=9 - 3440822“ - Kategorie: - * Wikipedia:Hauptseite - -Meine Werkzeuge - - * Anmelden / Benutzerkonto erstellen - -Namensräume - - * Hauptseite - * Diskussion - -Varianten - -Ansichten - - * Lesen - * Quelltext anzeigen - * Versionsgeschichte - -Aktionen - -Suche - - ____________________ (Submit) Volltext - -Navigation - - * Hauptseite - * Über Wikipedia - * Themenportale - * Von A bis Z - * Zufälliger Artikel - -Mitmachen - - * Neuen Artikel anlegen - * Autorenportal - * Hilfe - * Letzte Änderungen - * Kontakt - * Spenden - -Drucken/exportieren - - * Buch erstellen - * Als PDF herunterladen - * Druckversion - -Werkzeuge - - * Links auf diese Seite - * Änderungen an verlinkten Seiten - * Spezialseiten - * Permanenter Link - -In anderen Sprachen - - * Alemannisch - * العربية - * Boarisch - * Català - * Česky - * Dansk - * Ελληνικά - * English - * Español - * Français - * Nordfriisk - * Frysk - * हिन्दी - * Hrvatski - * Hornjoserbsce - * Magyar - * Italiano - * 日本語 - * Ripoarisch - * Latina - * Lëtzebuergesch - * Limburgs - * Plattdüütsch - * Nederlands - * ‪Norsk (bokmål)‬ - * Deitsch - * Polski - * Português - * Rumantsch - * Română - * Русский - * Slovenčina - * Slovenščina - * Shqip - * Српски / Srpski - * Seeltersk - * Svenska - * Türkçe - * Українська - * ייִדיש - * 中文 - - * Diese Seite wurde zuletzt am 9. September 2011 um 12:07 Uhr - geändert. - * Der Text ist unter der Lizenz „Creative Commons Attribution/Share - Alike“ verfügbar; zusätzliche Bedingungen können anwendbar sein. - Einzelheiten sind in den Nutzungsbedingungen beschrieben. - Wikipedia® ist eine eingetragene Marke der Wikimedia Foundation - Inc. - - * Datenschutz - * Über Wikipedia - * Impressum - * Mobile Ansicht - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/de/WINDOWS-1252/wikitop_de_WINDOWS-1252.txt b/src/tests/testdata/de/WINDOWS-1252/wikitop_de_WINDOWS-1252.txt deleted file mode 100644 index d8dc10f..0000000 --- a/src/tests/testdata/de/WINDOWS-1252/wikitop_de_WINDOWS-1252.txt +++ /dev/null @@ -1,219 +0,0 @@ - Wikipedia ist ein Projekt zum Aufbau einer Enzyklopdie aus freien - Inhalten in allen Sprachen der Welt. Jeder kann mit seinem Wissen - beitragen. Seit Mai 2001 sind so 1.301.685 Artikel in deutscher Sprache - entstanden. Gute Autorinnen und Autoren sind stets willkommen. - - Geographie Geschichte Gesellschaft Kunst und Kultur - Religion Sport Technik Wissenschaft - - Artikel nach Themen Alphabetischer Index Artikel nach Kategorien - Gesprochene Wikipedia - - Kontakt Presse Statistik Andere Sprachen Mentorenprogramm - - Wikipedia aktuell - - Bis zum 31. Oktober knnen Stimmen fr den Publikumspreis des - Schreibwettbewerbs abgegeben werden. - -Artikel des Tages - - Ku-ring-gai-Chase-Nationalpark - - Der Ku-ring-gai-Chase-Nationalpark ist ein Nationalpark bei Sydney in - Australien. Er wurde 1894 gegrndet und ist damit einer der ltesten - Nationalparks Australiens. Der Park umfasst etwa 150 km von Eukalyptus - dominiertes Buschland auf Sandstein-Hhenzgen sowie einige Rias, vom - Pazifischen Ozean berflutete Flusstler. Traditionelle Eigentmer des - Landes waren die Guringai, ein Stamm der Aborigines. Im Nationalpark - befinden sich viele Artefakte ihrer Besiedlung, unter anderem - Petroglyphen im Stil der Sydney-Felsgravuren. Ab 1834 siedelten Weie - im Gebiet; heute befindet sich mit Cottage Point aber nur noch ein - bewohnter Ort im Park. Fr die Bewohner Sydneys ist der Nationalpark - ein beliebtes Naherholungsgebiet mit jhrlich rund zwei Millionen - Besuchern. mehr - Archiv Weitere exzellente und lesenswerte Artikel RSS-Feed - RSS-Feed - -Was geschah am 21. Oktober? - - * Erzherzog Karl und Prinzessin Zita im Juli 1911 - 1911 In Schwarzau am Steinfeld heiraten Erzherzog Karl von - sterreich und Prinzessin Zita von Bourbon-Parma, von 1916 bis 1918 - sterreichs letztes Kaiserpaar. - * 1941 Massenerschieung von Kragujevac: Einheiten der deutschen - Wehrmacht ermorden mehr als 2300 serbische Zivilisten. - * 1971 Dem chilenischen Dichter Pablo Neruda wird der Nobelpreis - fr Literatur zuerkannt. - * 1986 Die Afrikanische Charta der Menschenrechte und der Rechte - der Vlker tritt in Kraft. - * 2006 In Danzig nimmt sich eine am Tag zuvor vor ihrer Klasse von - Mitschlern sexuell erniedrigte 14-jhrige Gymnasiastin das Leben. - - Weitere Ereignisse - -In den Nachrichten - - ETA Occupy Wall Street Trkisch-kurdischer Konflikt - __________________________________________________________________ - - Muammar al-Gaddafi (2009) - * Mit dem ersten Start einer russischen Sojus-Rakete vom europischen - Weltraumbahnhof Kourou sind die ersten beiden Satelliten des - Navigationssatellitensystems Galileo ins All gebracht worden. - * Muammar al-Gaddafi (Bild), der frhere Machthaber Libyens, ist in - seiner Heimatstadt Sirte erschossen worden. - * Der britische Autor Julian Barnes hat fr seinen Roman The Sense of - an Ending den renommierten Booker Prize erhalten. - - Weitere aktuelle Ereignisse Wikinews - -Krzlich Verstorbene - - * Jean Jlich (82), deutscher Widerstandskmpfer ( 19. Oktober) - * Bohdan Osadczuk (91), ukrainischer Autor ( 19. Oktober) - * Norman Corwin (101), US-amerikanischer Autor ( 18. Oktober) - - Weitere krzlich Verstorbene - -Schon gewusst? - - Westseite der Burg Friedestrom - * Die niederrheinische Burg Friedestrom in Zons ist sowohl Bau- als - auch Bodendenkmal. - * ber die Herkunft des Komponisten Sigismondo dIndias ist wenig - bekannt, aber er gilt als Wegbereiter der Barockmusik in Italien. - * Die USS Newport News (CA-148) war der letzte Kreuzer im aktiven - Dienst der United States Navy, der nur mit Geschtzen bewaffnet - war. - * Rosa Manus war ber 30 Jahre lang bis zum deutschen Einmarsch 1940 - eine der wichtigsten Leitfiguren der niederlndischen - Frauenbewegung. - - Weitere neue Artikel Frhere Schon-gewusst-Artikel - -Schwesterprojekte - - Meta-Wiki Koordination Commons Medien - Wiktionary Wrterbuch Wikibooks Lehrbcher - Wikiquote Zitate Wikiversity Lernplattform - Wikisource Quellen Wikinews Nachrichten - - Von - http://de.wikipedia.org/w/index.php?title=Wikipedia:Hauptseite&oldid=9 - 3440822 - Kategorie: - * Wikipedia:Hauptseite - -Meine Werkzeuge - - * Anmelden / Benutzerkonto erstellen - -Namensrume - - * Hauptseite - * Diskussion - -Varianten - -Ansichten - - * Lesen - * Quelltext anzeigen - * Versionsgeschichte - -Aktionen - -Suche - - ____________________ (Submit) Volltext - -Navigation - - * Hauptseite - * ber Wikipedia - * Themenportale - * Von A bis Z - * Zuflliger Artikel - -Mitmachen - - * Neuen Artikel anlegen - * Autorenportal - * Hilfe - * Letzte nderungen - * Kontakt - * Spenden - -Drucken/exportieren - - * Buch erstellen - * Als PDF herunterladen - * Druckversion - -Werkzeuge - - * Links auf diese Seite - * nderungen an verlinkten Seiten - * Spezialseiten - * Permanenter Link - -In anderen Sprachen - - * Alemannisch - * - * Boarisch - * Catal - * esky - * Dansk - * - * English - * Espaol - * Franais - * Nordfriisk - * Frysk - * - * Hrvatski - * Hornjoserbsce - * Magyar - * Italiano - * - * Ripoarisch - * Latina - * Ltzebuergesch - * Limburgs - * Plattdtsch - * Nederlands - * Norsk (bokml) - * Deitsch - * Polski - * Portugus - * Rumantsch - * Romn - * - * Slovenina - * Slovenina - * Shqip - * / Srpski - * Seeltersk - * Svenska - * Trke - * - * - * - - * Diese Seite wurde zuletzt am 9. September 2011 um 12:07 Uhr - gendert. - * Der Text ist unter der Lizenz Creative Commons Attribution/Share - Alike verfgbar; zustzliche Bedingungen knnen anwendbar sein. - Einzelheiten sind in den Nutzungsbedingungen beschrieben. - Wikipedia ist eine eingetragene Marke der Wikimedia Foundation - Inc. - - * Datenschutz - * ber Wikipedia - * Impressum - * Mobile Ansicht - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/de/iso-8859-1.txt b/src/tests/testdata/de/iso-8859-1.txt new file mode 100644 index 0000000..726a6c8 --- /dev/null +++ b/src/tests/testdata/de/iso-8859-1.txt @@ -0,0 +1,11 @@ +ISO 8859-1, genauer ISO/IEC 8859-1, auch bekannt als Latin-1, ist ein von der +ISO zuletzt 1998 aktualisierter Standard fr die Informationstechnik zur +Zeichenkodierung mit acht Bit und der erste Teil der Normenfamilie ISO/IEC 8859. + +Die mit sieben Bit kodierbaren Zeichen entsprechen US-ASCII mit fhrendem +Nullbit. Zustzlich zu den 95 darstellbaren ASCII-Zeichen (2016-7E16) kodiert +ISO 8859-1 96 weitere (A016-FF16), also insgesamt 191 von theoretisch mglichen +256 (= 28). Den Positionen 0016-1F16 und 7F16-9F16 sind in ISO/IEC 8859 und +damit ISO/IEC 8859-1 keine Zeichen zugewiesen. Die von der IANA definierte +Bezeichnung ISO-8859-1 (mit Bindestrich) steht fr die Kombination der Zeichen +dieser Norm mit nicht darstellbaren Steuerzeichen gem ISO/IEC 6429. diff --git a/src/tests/testdata/de/windows-1252.txt b/src/tests/testdata/de/windows-1252.txt new file mode 100644 index 0000000..7c51f46 --- /dev/null +++ b/src/tests/testdata/de/windows-1252.txt @@ -0,0 +1,11 @@ +ISO 8859-1, genauer ISO/IEC 8859-1, auch bekannt als Latin-1, ist ein von der +ISO zuletzt 1998 aktualisierter Standard fr die Informationstechnik zur +Zeichenkodierung mit acht Bit und der erste Teil der Normenfamilie ISO/IEC 8859. + +Die mit sieben Bit kodierbaren Zeichen entsprechen US-ASCII mit fhrendem +Nullbit. Zustzlich zu den 95 darstellbaren ASCII-Zeichen (20167E16) kodiert +ISO 8859-1 96 weitere (A016FF16), also insgesamt 191 von theoretisch mglichen +256 (= 28). Den Positionen 00161F16 und 7F169F16 sind in ISO/IEC 8859 und +damit ISO/IEC 8859-1 keine Zeichen zugewiesen. Die von der IANA definierte +Bezeichnung ISO-8859-1 (mit Bindestrich) steht fr die Kombination der Zeichen +dieser Norm mit nicht darstellbaren Steuerzeichen gem ISO/IEC 6429. diff --git a/src/tests/testdata/dk/UTF-8/wikitop_dk_UTF-8.txt b/src/tests/testdata/dk/UTF-8/wikitop_dk_UTF-8.txt deleted file mode 100644 index 4add81f..0000000 --- a/src/tests/testdata/dk/UTF-8/wikitop_dk_UTF-8.txt +++ /dev/null @@ -1,332 +0,0 @@ - Der findes nu over 19 millioner artikler i Wikipedia, hvoraf 156.678 er - på dansk. - Skribentforside • Hjælp • A–Å • Projekter • Bekendtgørelser • - Ambassaden - Embassy • Wikipedia Mobil • Kontakt os - Emblem-star.svg - Ugens artikel - Winston Churchill, 1942 - - Sir Winston Leonard Spencer Churchill (1874 - 1965) var en britisk - politiker og statsmand, som især er kendt for sit lederskab af - Storbritannien under 2. verdenskrig. Han var premierminister i - Storbritannien fra 1940 til 1945 og igen fra 1951 til 1955. Han - betragtes i vide kredse som en af de største ledere i krigstid. - - Ved siden af sit politiske arbejde var han forfatter til 37 bøger - fordelt på i alt 58 bind, og han nåede at male over 500 malerier. Som - forfatter skrev han navnlig populærhistoriske værker samt - oversigtsværker over samtiden. Han er den eneste britiske - premierminister, som har modtaget Nobelprisen i litteratur, og han var - den første, som blev udnævnt til æresborger i De forenede Stater. - - Churchill blev født ind i en aristokratisk familie. Hans far Lord - Randolph Churchill, som var efterkommer af hertugen af Marlborough, var - en karismatisk politiker, som bl.a. var britisk finansminister. Hans - mor, Jenny Jerome, stammede fra en velhavende amerikansk familie. - Da det konservative parti havde tabt parlamentsvalget i 1945, blev han - leder af oppositionen. I 1951 dannede han igen regering, og han beholdt - posten indtil han trak sig tilbage i 1955. Da han døde i 1965 tilstod - dronningen ham en statsbegravelse, som tiltrak et af de største antal - af verdens statsmænd, som nogensinde har været samlet. Læs mere - - Seneste tre uger: Telefonkiosk — Svalbard Lufthavn, Longyear — Café - Bernina - - Mere om Ugens artikel • Arkiv - Norway-sweden2.svg - Dagens skandinaviske artikel - Härskarringen ges till den som i spelet dödar Gollum. - - Slaget om Midgård 2 – Ringarnas Herre är ett realtidsstrategi-datorspel - från 2006 utvecklat och utgivet av Electronic Arts. Det baseras på - fantasyromanen Sagan om Ringen av Tolkien och dess ombearbetning till - filmtrilogin. Spelet är efterföljare till Electronic Arts spel Slaget - om Midgård – Härskarringen från 2004. - - Historien i Slaget om Midgård 2 – Ringarnas Herre är uppdelad i två - spelvärldar, den Goda och den Onda. Den Goda spelvärlden fokuserar på - Glorfindel, en alv som varnas om en kommande attack mot alvhelgedomen - Vattnadal. Med hjälp från dvärgar och andra styrkor på den goda sidan - försöker alverna att eliminera Sauron och dennes armé för att - återställa freden i Midgård. I den Onda spelvärlden sänder Sauron ut - sitt språkrör och sina Nazgûler för att samla ihop vilda vättar. Med - armén samlad kan Sauron fortsätta med sin plan att krossa de - återstående Goda styrkorna i norr. Läs mer - Mere om Dagens skandinaviske artikel • Skanwiki - Searchtool right.svg - Fremhævede artikler - - Fremragende artikler - -De nyeste fremragende artikler - - Islands Brygge • Jorden • Månen • Nordisk religion • The Final Cut - Gode artikler - -De nyeste gode artikler - - Terrorangrebet den 11. september 2001 • Svalbard Lufthavn, Longyear • - Winston Churchill - Lovende artikler - -De nyeste lovende artikler - - 12. til 21. etape af Giro d'Italia 2009 • God Put a Smile upon Your - Face • The Scientist - Rediger • Mere fremhævet indhold - Gnome-fs-blockdev.svg - Kategorier - Naturvidenskab - -Naturvidenskab - - Astronomi • Biologi • Datalogi • Fysik • Geografi • Geologi • Kemi • - Logik • Lægevidenskab • Matematik - Humaniora - -Humaniora og samfundsvidenskab - - Antropologi • Arkitektur • Filosofi • Historie • Jura • Kommunikation • - Psykologi • Pædagogik • Sociologi • Sprogforskning • Statskundskab • - Teologi • Økonomi - Teknik - -Teknik - - Elektronik • Energi • Industri • Jordbrug • Mekanik • Militær • - Robotter • Sundhedsvæsen • Transport - Kultur - -Kultur - - Dans • Film • Hobby • Kunst • Litteratur • Mad og drikke • Mode • - Musik • Mytologi • Personer • Politik • Religion • Sport • Teater • - Traditioner • Turisme • Underholdning - Flere kategorier - Internet-web-browser.svg - Aktuelle begivenheder - * Hungersnøden på Afrikas Horn • Oprøret i Libyen 2011 - ______________________________________________________________ - - Muammar al-Gaddafi at the AU summit.jpg - * Libyens statsleder Muammar al-Gaddafi (billedet) bliver dræbt under - oprøret i Libyen. - * Danmarks fodboldlandshold kvalificerer sig til EM-slutrunden 2012 - efter sejr i afgørende kamp over Portugal med 2-1 på mål af Michael - Krohn-Dehli og Nicklas Bendtner. - * Ellen Johnson Sirleaf, Leymah Gbowee, og Tawakkul Karman deler - Nobels fredspris for deres ikke-voldelige kamp for kvinders - rettigheder i Afrika. - * Apples bestyrelsesformand Steve Jobs dør i en alder af 56 år. - - Rediger • Flere aktuelle begivenheder - Office-calendar.svg - I dag - - Den 21. oktober: - Den ny Lillebæltsbro - * 1805 - Admiral Lord Nelson dør under Slaget ved Trafalgar. - Englænderne vinder dette vigtige slag mod Napoleons kombinerede - franske og spanske flåder ud for Kap Trafalgar, sydvest for - Spanien. - * 1807 - Englænderne sejler af sted med den danske flåde: 15 - linjeskibe, 15 fregatter, 8 brigger og 31 andre, mindre fartøjer. - Danmark tvinges ind i Napoleonskrigene på fransk side. - * 1970 - Den nye Lillebæltsbro indvies af Frederik 9. - * - - Rediger • Alle månedens dage - Help-browser.svg - Vidste du at... - Bosphore - Pont du conquerant.jpg - - Fra Wikipedias nyeste artikler… - - * ...Fatih Sultan Mehmet Broen (billedet) i Istanbul er opkaldt efter - den osmanniske sultan Mehmed 2.? - - * ...den svenske operasanger Ingvar Wixell (1931-2011) blev regnet - blandt 1900-tallets betydeligste barytoner? - - * ... albummet Under the Mistletoe bliver den canadiske musiker - Justin Biebers andet studiealbum? - - Rediger • Flere vidste du at... - Gravestone icon2.svg - Nyligt afdøde - * 20. oktober - Muammar al-Gaddafi, libysk statsleder (født 1942). - * 17. oktober - Poul Glargaard, dansk skuespiller (født 1942). - * 16. oktober - Henning Sjöström, svensk advokat (født 1922). - - Rediger • Flere nyligt afdøde - Image-x-generic.svg - Dagens billede - - Schimpanse, Pan troglodytes 3.JPG Chimpanse (Pan troglodytes). - - Rediger • Arkiv - Wikipedia-logo.png - Søsterprojekter - - Wikipedia ejes af paraplyorganisationen Wikimedia Foundation, som - driver andre flersproglige og frie projekter, hvor alle kan bidrage. - Meta-Wiki - Wiktionary - Wikisource - Wikiquote - Wikibooks - Wikinews - Wikispecies - Commons - Meta-Wiki Wiktionary Wikisource Wikiquote Wikibooks Wikinews - Wikispecies Commons - Om Wikiprojekterne Flersproget ordbog Kildemateriale Citatsamling - Gratis bøger og manualer Nyhedstjeneste Videnskabelig artsoversigt - Fildeling af billeder og lyd - Mere om Wikimedia • Serverstatus - Preferences-desktop-locale.svg - Wikipedia på andre sprog - - Føroyskt | Íslenska | Kalaallisut | Norsk (bokmål) | Nynorsk | - Suomi | Svenska - - * Flere end 300.000 artikler: - Català · Deutsch · English · Español · Français · Italiano · - Nederlands · 日本語 · Norsk (bokmål) · Polski · Português · Русский · - Svenska · Українська · 中文 - - * Flere end 125.000 artikler: - العربية · Bahasa Indonesia · Čeština · Dansk · Esperanto · فارسی · - 한국어 · Lietuvių · Magyar · Română · Slovenčina · Српски / Srpski · - Suomi · Tiếng Việt · Türkçe - - * Flere end 50.000 artikler: Azərbaycan · Bahasa Melayu · Български · - Eesti · Ελληνικά · English (simple) · Euskara · Galego · עברית · - हिन्दी · Hrvatski · ქართული · Krèyol ayisyen · Latina · नेपाल भाषा - · Norsk (nynorsk) · Slovenščina · Tagalog · ไทย · Volapük - - Komplet liste - Wikimedia-logo.svg - Donationer - - Moderselskabet Wikimedia Foundation er uafhængigt af alle interesser og - behøver derfor økonomisk støtte fra læsere og brugere for at holde - driften i gang. Giv et bidrag til Wikimedia og vær med til at sikre - udbygningen af de servere, som Wikipedia og søsterprojekterne afvikles - på. - Støt Wikipedia - -Til toppen af siden - - Hentet fra - "http://da.wikipedia.org/w/index.php?title=Forside&oldid=2692411" - Kategori: - * Wikipedia - -Personlige værktøjer - - * Opret en konto eller log på - -Navnerum - - * Artikel - * Diskussion - -Varianter - -Visninger - - * Vis - * Vis kilden - * Se historik - -Handlinger - -Søg - - ____________________ (Submit) Søg - -Navigation - - * Forside - * Kategorier - * Fremhævet indhold - * Tilfældig artikel - * Aktuelle begivenheder - -Deltagelse - - * Velkommen - * Skribentforside - * Projekter - * Seneste ændringer - * Hjælp - -Værktøjer - - * Hvad henviser hertil - * Relaterede ændringer - * Specialsider - * Permanent link - * Citér denne artikel - -Organisation - - * Donation - * Kontakt Wikipedia - * Wikimedia Danmark - -Udskriv/eksportér - - * Lav en bog - * Download som PDF - * Udskriftsvenlig udgave - -Andre sprog - - * العربية - * Žemaitėška - * Bosanski - * Català - * Нохчийн - * Словѣ́ньскъ / ⰔⰎⰑⰂⰡⰐⰠⰔⰍⰟ - * Deutsch - * English - * Español - * Suomi - * Føroyskt - * Français - * Hrvatski - * Íslenska - * Italiano - * 日本語 - * Kalaallisut - * Lietuvių - * Latviešu - * Nederlands - * ‪Norsk (nynorsk)‬ - * ‪Norsk (bokmål)‬ - * Polski - * Português - * Română - * Русский - * Srpskohrvatski / Српскохрватски - * Simple English - * Српски / Srpski - * Svenska - * Türkçe - * Татарча/Tatarça - * 中文 - - * Denne side blev senest ændret den 27. december 2008 kl. 22:01. - * Tekst er tilgængelig under Creative Commons Navngivelse/Del på - samme vilkår 3.0; yderligere betingelser kan være gældende. Se - brugsbetingelserne for flere oplysninger. - - * Behandling af personlige oplysninger - * Om Wikipedia - * Forbehold - * Mobil visning - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/dk/WINDOWS-1252/wikitop_dk_WINDOWS-1252.txt b/src/tests/testdata/dk/WINDOWS-1252/wikitop_dk_WINDOWS-1252.txt deleted file mode 100644 index 239ea83..0000000 --- a/src/tests/testdata/dk/WINDOWS-1252/wikitop_dk_WINDOWS-1252.txt +++ /dev/null @@ -1,324 +0,0 @@ - - politiker og statsmand, som isr er kendt for sit lederskab af - Storbritannien under 2. verdenskrig. Han var premierminister i - Storbritannien fra 1940 til 1945 og igen fra 1951 til 1955. Han - betragtes i vide kredse som en af de strste ledere i krigstid. - - Ved siden af sit politiske arbejde var han forfatter til 37 bger - fordelt p i alt 58 bind, og han nede at male over 500 malerier. Som - forfatter skrev han navnlig populrhistoriske vrker samt - oversigtsvrker over samtiden. Han er den eneste britiske - premierminister, som har modtaget Nobelprisen i litteratur, og han var - den frste, som blev udnvnt til resborger i De forenede Stater. - - Churchill blev fdt ind i en aristokratisk familie. Hans far Lord - Randolph Churchill, som var efterkommer af hertugen af Marlborough, var - en karismatisk politiker, som bl.a. var britisk finansminister. Hans - mor, Jenny Jerome, stammede fra en velhavende amerikansk familie. - Da det konservative parti havde tabt parlamentsvalget i 1945, blev han - leder af oppositionen. I 1951 dannede han igen regering, og han beholdt - posten indtil han trak sig tilbage i 1955. Da han dde i 1965 tilstod - dronningen ham en statsbegravelse, som tiltrak et af de strste antal - af verdens statsmnd, som nogensinde har vret samlet. Ls mere - - Seneste tre uger: Telefonkiosk Svalbard Lufthavn, Longyear Caf - Bernina - - Mere om Ugens artikel Arkiv - Norway-sweden2.svg - Dagens skandinaviske artikel - Hrskarringen ges till den som i spelet ddar Gollum. - - Slaget om Midgrd 2 Ringarnas Herre r ett realtidsstrategi-datorspel - frn 2006 utvecklat och utgivet av Electronic Arts. Det baseras p - fantasyromanen Sagan om Ringen av Tolkien och dess ombearbetning till - filmtrilogin. Spelet r efterfljare till Electronic Arts spel Slaget - om Midgrd Hrskarringen frn 2004. - - Historien i Slaget om Midgrd 2 Ringarnas Herre r uppdelad i tv - spelvrldar, den Goda och den Onda. Den Goda spelvrlden fokuserar p - Glorfindel, en alv som varnas om en kommande attack mot alvhelgedomen - Vattnadal. Med hjlp frn dvrgar och andra styrkor p den goda sidan - frsker alverna att eliminera Sauron och dennes arm fr att - terstlla freden i Midgrd. I den Onda spelvrlden snder Sauron ut - sitt sprkrr och sina Nazgler fr att samla ihop vilda vttar. Med - armn samlad kan Sauron fortstta med sin plan att krossa de - terstende Goda styrkorna i norr. Ls mer - Mere om Dagens skandinaviske artikel Skanwiki - Searchtool right.svg - Fremhvede artikler - - Fremragende artikler - -De nyeste fremragende artikler - - Islands Brygge Jorden Mnen Nordisk religion The Final Cut - Gode artikler - -De nyeste gode artikler - - Terrorangrebet den 11. september 2001 Svalbard Lufthavn, Longyear - Winston Churchill - Lovende artikler - -De nyeste lovende artikler - - 12. til 21. etape af Giro d'Italia 2009 God Put a Smile upon Your - Face The Scientist - Rediger Mere fremhvet indhold - Gnome-fs-blockdev.svg - Kategorier - Naturvidenskab - -Naturvidenskab - - Astronomi Biologi Datalogi Fysik Geografi Geologi Kemi - Logik Lgevidenskab Matematik - Humaniora - -Humaniora og samfundsvidenskab - - Antropologi Arkitektur Filosofi Historie Jura Kommunikation - Psykologi Pdagogik Sociologi Sprogforskning Statskundskab - Teologi konomi - Teknik - -Teknik - - Elektronik Energi Industri Jordbrug Mekanik Militr - Robotter Sundhedsvsen Transport - Kultur - -Kultur - - Dans Film Hobby Kunst Litteratur Mad og drikke Mode - Musik Mytologi Personer Politik Religion Sport Teater - Traditioner Turisme Underholdning - Flere kategorier - Internet-web-browser.svg - Aktuelle begivenheder - * Hungersnden p Afrikas Horn Oprret i Libyen 2011 - ______________________________________________________________ - - Muammar al-Gaddafi at the AU summit.jpg - * Libyens statsleder Muammar al-Gaddafi (billedet) bliver drbt under - oprret i Libyen. - * Danmarks fodboldlandshold kvalificerer sig til EM-slutrunden 2012 - efter sejr i afgrende kamp over Portugal med 2-1 p ml af Michael - Krohn-Dehli og Nicklas Bendtner. - * Ellen Johnson Sirleaf, Leymah Gbowee, og Tawakkul Karman deler - Nobels fredspris for deres ikke-voldelige kamp for kvinders - rettigheder i Afrika. - * Apples bestyrelsesformand Steve Jobs dr i en alder af 56 r. - - Rediger Flere aktuelle begivenheder - Office-calendar.svg - I dag - - Den 21. oktober: - Den ny Lillebltsbro - * 1805 - Admiral Lord Nelson dr under Slaget ved Trafalgar. - Englnderne vinder dette vigtige slag mod Napoleons kombinerede - franske og spanske flder ud for Kap Trafalgar, sydvest for - Spanien. - * 1807 - Englnderne sejler af sted med den danske flde: 15 - linjeskibe, 15 fregatter, 8 brigger og 31 andre, mindre fartjer. - Danmark tvinges ind i Napoleonskrigene p fransk side. - * 1970 - Den nye Lillebltsbro indvies af Frederik 9. - * - - Rediger Alle mnedens dage - Help-browser.svg - Vidste du at... - Bosphore - Pont du conquerant.jpg - - Fra Wikipedias nyeste artikler - - * ...Fatih Sultan Mehmet Broen (billedet) i Istanbul er opkaldt efter - den osmanniske sultan Mehmed 2.? - - * ...den svenske operasanger Ingvar Wixell (1931-2011) blev regnet - blandt 1900-tallets betydeligste barytoner? - - * ... albummet Under the Mistletoe bliver den canadiske musiker - Justin Biebers andet studiealbum? - - Rediger Flere vidste du at... - Gravestone icon2.svg - Nyligt afdde - * 20. oktober - Muammar al-Gaddafi, libysk statsleder (fdt 1942). - * 17. oktober - Poul Glargaard, dansk skuespiller (fdt 1942). - * 16. oktober - Henning Sjstrm, svensk advokat (fdt 1922). - - Rediger Flere nyligt afdde - Image-x-generic.svg - Dagens billede - - Schimpanse, Pan troglodytes 3.JPG Chimpanse (Pan troglodytes). - - Rediger Arkiv - Wikipedia-logo.png - Ssterprojekter - - Wikipedia ejes af paraplyorganisationen Wikimedia Foundation, som - driver andre flersproglige og frie projekter, hvor alle kan bidrage. - Meta-Wiki - Wiktionary - Wikisource - Wikiquote - Wikibooks - Wikinews - Wikispecies - Commons - Meta-Wiki Wiktionary Wikisource Wikiquote Wikibooks Wikinews - Wikispecies Commons - Om Wikiprojekterne Flersproget ordbog Kildemateriale Citatsamling - Gratis bger og manualer Nyhedstjeneste Videnskabelig artsoversigt - Fildeling af billeder og lyd - Mere om Wikimedia Serverstatus - Preferences-desktop-locale.svg - Wikipedia p andre sprog - - Froyskt | slenska | Kalaallisut | Norsk (bokml) | Nynorsk | - Suomi | Svenska - - * Flere end 300.000 artikler: - Catal Deutsch English Espaol Franais Italiano - Nederlands Norsk (bokml) Polski Portugus - Svenska - - * Flere end 125.000 artikler: - Bahasa Indonesia etina Dansk Esperanto - Lietuvi Magyar Romn Slovenina / Srpski - Suomi Ting Vit Trke - - * Flere end 50.000 artikler: Azrbaycan Bahasa Melayu - Eesti English (simple) Euskara Galego - Hrvatski Kryol ayisyen Latina - Norsk (nynorsk) Slovenina Tagalog Volapk - - Komplet liste - Wikimedia-logo.svg - Donationer - - Moderselskabet Wikimedia Foundation er uafhngigt af alle interesser og - behver derfor konomisk sttte fra lsere og brugere for at holde - driften i gang. Giv et bidrag til Wikimedia og vr med til at sikre - udbygningen af de servere, som Wikipedia og ssterprojekterne afvikles - p. - Stt Wikipedia - -Til toppen af siden - - Hentet fra - "http://da.wikipedia.org/w/index.php?title=Forside&oldid=2692411" - Kategori: - * Wikipedia - -Personlige vrktjer - - * Opret en konto eller log p - -Navnerum - - * Artikel - * Diskussion - -Varianter - -Visninger - - * Vis - * Vis kilden - * Se historik - -Handlinger - -Sg - - ____________________ (Submit) Sg - -Navigation - - * Forside - * Kategorier - * Fremhvet indhold - * Tilfldig artikel - * Aktuelle begivenheder - -Deltagelse - - * Velkommen - * Skribentforside - * Projekter - * Seneste ndringer - * Hjlp - -Vrktjer - - * Hvad henviser hertil - * Relaterede ndringer - * Specialsider - * Permanent link - * Citr denne artikel - -Organisation - - * Donation - * Kontakt Wikipedia - * Wikimedia Danmark - -Udskriv/eksportr - - * Lav en bog - * Download som PDF - * Udskriftsvenlig udgave - -Andre sprog - - * - * emaitka - * Bosanski - * Catal - * - * / - * Deutsch - * English - * Espaol - * Suomi - * Froyskt - * Franais - * Hrvatski - * slenska - * Italiano - * - * Kalaallisut - * Lietuvi - * Latvieu - * Nederlands - * Norsk (nynorsk) - * Norsk (bokml) - * Polski - * Portugus - * Romn - * - * Srpskohrvatski / - * Simple English - * / Srpski - * Svenska - * Trke - * /Tatara - * - - * Denne side blev senest ndret den 27. december 2008 kl. 22:01. - * Tekst er tilgngelig under Creative Commons Navngivelse/Del p - samme vilkr 3.0; yderligere betingelser kan vre gldende. Se - brugsbetingelserne for flere oplysninger. - - * Behandling af personlige oplysninger - * Om Wikipedia - * Forbehold - * Mobil visning - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/el/.WINDOWS-1253/README b/src/tests/testdata/el/.WINDOWS-1253/README deleted file mode 100644 index 39ad79f..0000000 --- a/src/tests/testdata/el/.WINDOWS-1253/README +++ /dev/null @@ -1 +0,0 @@ -Impossible to auto-discriminate from ISO-8859-7 ? diff --git a/src/tests/testdata/el/.WINDOWS-1253/wikitop_el_WINDOWS-1253.txt b/src/tests/testdata/el/.WINDOWS-1253/wikitop_el_WINDOWS-1253.txt deleted file mode 100644 index f9572d3..0000000 --- a/src/tests/testdata/el/.WINDOWS-1253/wikitop_el_WINDOWS-1253.txt +++ /dev/null @@ -1,239 +0,0 @@ -: - - , - : , - - , - . - - 65.691 . - - - - . - - - Nuvola filesystems services.svg - - 1888 - - 1950 1990 - . - 1821. - - . - - - 1821. - 1888 - .(...) - ... - Gnome globe current event.svg - * . - * . - * - . - * - - . - - ... - Nuvola apps date.svg - 21 - - * 1944 - HMAS Australia . - 30 , . - (' ) - * 1972 - NAMC YS-11A - . 37 , 19 - . - - , . - - Nuvola apps filetypes.svg ... - * - , - ; - * - ; - * 19 - ; - * - ; - - - - Nuvola apps email.png - - . - , . , - . - - . ... ! - Nuvola apps kpdf.png - - - Nuvola apps kcoloredit.svg - - - - - - Nuvola apps kalzium.png - - - - - - Nuvola apps edu phi.png - - - - - Gnome-globe.svg - - - - - Nuvola apps bookcase.png - - - - ... - - - 1 2 3 4 5 6 7 8 9 0 - Information icon.svg - Wikipedia-logo.png - - - - - - Nuvola apps kuser.svg - (FAQ) - - - - Monobook icon.svg - - - - - - - - Wikimedia Foundation, - : - - Wikisource-logo.svg - - - - - Wiktprintable without text.svg - - - - - Wikinews-logo.svg - - - - - Commons-logo.svg - - Commons - - - Wikibooks-logo.svg - - - - - Wikiquote-logo.svg - - - - - Wikiversity-logo.svg - - - - - Wikispecies-logo.svg - - Wikispecies - - - Wikimedia Community Logo.svg - - Meta-Wiki - Wikimedia - - - - - - * / - - - - * - * - - - - - - * - * - * - - - - - - ____________________ (Submit) - - - - * - * - * - * - * - - - - * - * - * - * - * - * - -/ - - * - * PDF - * - - - - * - * - * - * - - - - * 20:32, 8 2011. - * Creative Commons - Attribution/Share-Alike License - . . - - * - * - * - * - diff --git a/src/tests/testdata/el/ISO-8859-7/wikitop_el_ISO-8859-7.txt b/src/tests/testdata/el/ISO-8859-7/wikitop_el_ISO-8859-7.txt deleted file mode 100644 index 84c521c..0000000 --- a/src/tests/testdata/el/ISO-8859-7/wikitop_el_ISO-8859-7.txt +++ /dev/null @@ -1,226 +0,0 @@ - , - : , - - , - . - - 65.691 . - - - - . - - - - - 1888 - - 1950 1990 - . - 1821. - - . - - - 1821. - 1888 - .(...) - ... - Gnome globe current event.svg - * . - * . - * - . - * - - . - - ... - Nuvola apps date.svg - 21 - - * 1944 - HMAS Australia . - 30 , . - (' ) - * 1972 - NAMC YS-11A - . 37 , 19 - . - - , . - - Nuvola apps filetypes.svg ... - * - , - ; - * - ; - * 19 - ; - * - ; - - - - Nuvola apps email.png - - . - , . , - . - - . ... ! - Nuvola apps kpdf.png - - - Nuvola apps kcoloredit.svg - - - - - - Nuvola apps kalzium.png - - - - - - Nuvola apps edu phi.png - - - - - Gnome-globe.svg - - - - - Nuvola apps bookcase.png - - - - ... - - - 1 2 3 4 5 6 7 8 9 0 - Information icon.svg - Wikipedia-logo.png - - - - - - Nuvola apps kuser.svg - (FAQ) - - - - Monobook icon.svg - - - - - - - - Wikimedia Foundation, - : - - - - - - - - - - - - - - - - - - Wikiquote-logo.svg - - - - - Wikiversity-logo.svg - - - - - Wikispecies-logo.svg - - Wikispecies - - - Wikimedia Community Logo.svg - - Meta-Wiki - Wikimedia - - - - - - * / - - - - * - * - - - - - - * - * - * - - - - - - ____________________ (Submit) - - - - * - * - * - * - * - - - - * - * - * - * - * - * - -/ - - * - * PDF - * - - - - * - * - * - * - - - * 20:32, 8 2011. - * Creative Commons - Attribution/Share-Alike License - . . - - * - * - * - * diff --git a/src/tests/testdata/el/UTF-8/wikitop_el_UTF-8.txt b/src/tests/testdata/el/UTF-8/wikitop_el_UTF-8.txt deleted file mode 100644 index 5ab4960..0000000 --- a/src/tests/testdata/el/UTF-8/wikitop_el_UTF-8.txt +++ /dev/null @@ -1,239 +0,0 @@ -Πύλη:Κύρια - - Από τη Βικιπαίδεια, την ελεύθερη εγκυκλοπαίδεια - Μετάβαση σε: πλοήγηση, αναζήτηση - - Καλώς ήλθατε στη Βικιπαίδεια, - την ελεύθερη εγκυκλοπαίδεια που γράφουμε συμμετοχικά. - Η ελληνική έκδοση της εγκυκλοπαίδειας περιλαμβάνει αυτή τη στιγμή - 65.691 άρθρα. - - Μάθετε περισσότερα για την εγκυκλοπαίδεια διαβάζοντας απαντήσεις σε - συνηθισμένες ερωτήσεις και τη σελίδα βοήθειας για τη χρήση και το έργο - αυτού του εγχειρήματος. - Έκδοση για κινητά και υπολογιστές χειρός - - Nuvola filesystems services.svg Επιλεγμένο άρθρο - Η Ιστορία της Πόλεως των Πατρών είναι ιστορικό βιβλίο του Στέφανου - Θωμόπουλου που εκδόθηκε για πρώτη φορά το 1888 στην Αθήνα και - επανεκδόθηκε από τον Κώστα Τριανταφύλλου με νέο υλικό που είχε - προσθέσει ο συγγραφέας το 1950 αλλά και το 1990 με προσθήκη στοιχείων - και σχολίων από τον ιστορικό Βασίλη Λάζαρη. Καλύπτει την ιστορία της - πόλης από τα αρχαία χρόνια μέχρι την Ελληνική Επανάσταση του 1821. Το - βιβλίο και οι επανεκδόσεις του έχουν χρησιμοποιηθεί ως βιβλιογραφία και - είναι έργα αναφοράς για την ιστορία της Πάτρας. - - Ο ολοκληρωμένος τίτλος του βιβλίου είναι Ιστορία της Πόλεως Πατρών από - των αρχαιοτάτων χρόνων μέχρι του 1821. Εκδόθηκε από το βασιλικό - τυπογραφείο Ιγγλέση στην Αθήνα το 1888 και αποτελεί το πρώτο έργο - τέτοιας εμβέλειας για την ιστορία μιας ελληνικής πόλης.(περισσότερα...) - Άλλα αξιόλογα άρθρα... - Gnome globe current event.svg Τρέχοντα γεγονότα - * Νέος Πρωθυπουργός του Νεπάλ εκλέγεται ο Μπαμπουράμ Μπαταράι. - * Παραιτείται από την πρωθυπουργία της Ιαπωνίας ο Ναότο Καν. - * Οι αντάρτες της Λιβύης καταλαμβάνουν την Τρίπολη και τερματίζεται - το καθεστώς του Μοαμάρ Καντάφι. - * Ο Μανουέλ Πίντο ντα Κόστα και ο Ζόρζε Κάρλος Φονσέκα κερδίζουν τις - προεδρικές εκλογές στις πρώην πορτογαλικές αποικίες Σάο Τομέ και - Πρίνσιπε και Πράσινο Ακρωτήριο αντίστοιχα. - - Περισσότερα τρέχοντα γεγονότα... - Nuvola apps date.svg Ημερολόγιο - 21 Οκτωβρίου - - * 1944 - Το HMAS Australia χτυπήθηκε κατά την πρώτη επίθεση Καμικάζι. - 30 άντρες σκοτώθηκαν, μεταξύ των οποίων και ο διοικητής του πλοίου. - (Β' Παγκόσμιος Πόλεμος) - * 1972 - Ένα NAMC YS-11A της Ολυμπιακής Αεροπορίας συνετρίβη στις - ακτές της Βούλας Αττικής. 37 από τους επιβαίνοντες πνίγηκαν, ενώ 19 - κατάφεραν να σωθούν φτάνοντας στην ακτή. - -Περισσότερα για αυτό τον μήνα, αυτή την ημέρα. - - Nuvola apps filetypes.svg Γνωρίζετε ότι... - * ο Αρτούρο Πέρεθ-Ρεβέρτε είναι Ισπανός συγγραφέας και δημοσιογράφος, - μέλος της Βασιλικής Ακαδημίας των Γραμμάτων; - * η Ντέιρντρε αποτελεί ίσως την τραγικότερη φιγούρα της ιραλνδικής - μυθολογίας; - * ο Κανό Χογκάι ήταν Ιάπωνας καλλιτέχνης του 19ου και ένας από τους - τελευταίους ζωγράφους της σχολής σχολής Κανό; - * η Φλόρενς Νάιτινγκεϊλ ήταν πρωτοπόρος της σύγχρονης νοσηλευτικής - αλλά και διακεκριμένη στατιστικός; - - Περισσότερα - - Nuvola apps email.png Πρόσκληση συμμετοχής - Η ελληνική Βικιπαίδεια χρειάζεται την εθελοντική σας συνεισφορά για να - αναπτυχθεί. Αν βρήκατε κάποιο άρθρο που σας ενδιαφέρει αλλά είναι - ανεπαρκές, συμπληρώστε το. Αν γνωρίζετε καλά κάποιο θέμα, μπορείτε να - ξεκινήσετε ένα νέο άρθρο από την αρχή. Ακόμα και απευθείας μεταφράσεις - άρθρων από εκδόσεις της Βικιπαίδειας σε άλλες γλώσσες είναι - καλοδεχούμενες. Διαβάστε τις σχετικές οδηγίες και ... καλό γράψιμο! - Nuvola apps kpdf.png Εγκυκλοπαίδεια - Θεματικός κατάλογος - - Nuvola apps kcoloredit.svg - Πολιτισμός και Τέχνες - - Ζωγραφική · Θέατρο · Κινηματογράφος · Λογοτεχνία · Μουσική · - Αρχιτεκτονική · Γλυπτική · Αθλητισμός · Μυθολογία · Θρησκεία - - Nuvola apps kalzium.png - Θετικές και Φυσικές Επιστήμες - - Αστρονομία · Βιολογία · Γεωλογία · Διάστημα · Υπολογιστές · Μαθηματικά - · Τεχνολογία · Φυσική · Χημεία · Ιατρική - - Nuvola apps edu phi.png - Φιλοσοφία και Κοινωνικές Επιστήμες - - Ανθρωπολογία ·Αρχαιολογία · Γλώσσα · Οικονομία · Φιλοσοφία · Ψυχολογία - - Gnome-globe.svg - Γεωγραφία - - Ασία · Αφρική · Αμερική · Ευρώπη · Ωκεανία · Πόλεις · Χώρες · Θάλασσες - - Nuvola apps bookcase.png - Ιστορία - - Αρχαιότητα · Μεσαίωνας · Νεότεροι χρόνοι · Πόλεμος - Περισσότερα θέματα... - Αλφαβητικός κατάλογος - Α Ά Β Γ Δ Ε Έ Ζ Η Ή Θ Ι Ί Κ Λ Μ Ν Ξ Ο Ό Π Ρ Σ Τ Υ Ύ Φ Χ Ψ Ω Ώ - 1 2 3 4 5 6 7 8 9 0 - Information icon.svg Βικιπαίδεια - Wikipedia-logo.png - Βικιπαίδεια - - Βοήθεια · Άρθρα προς επιμέλεια · Άρθρα προς μετάφραση · Πνευματικά - δικαιώματα · Άρθρα προς επέκταση - Κοινότητα - Nuvola apps kuser.svg - Αγορά · Συχνές ερωτήσεις (FAQ) · Πολιτική · Πολιτική διαγραφής · - Αμμοδόχος · Προτεινόμενα θέματα για νέα άρθρα · Άρθρα ζωτικής σημασίας - · Νέα άρθρα · Διαχειριστές - Οδηγίες για νέους χρήστες - Monobook icon.svg - Οδηγός για νέους χρήστες · Τι είναι η Βικιπαίδεια · Τι δεν είναι η - Βικιπαίδεια · Απλοί κανόνες · Ουδετερότητα · Εικόνες · Πώς να - επεξεργαστείτε μια σελίδα · Οδηγός μεταφράσεων · Επιφόρτωση αρχείου - -Άλλα εγχειρήματα - - Η λειτουργία της Βικιπαίδειας υποστηρίζεται από τον μη κερδοσκοπικό - οργανισμό Wikimedia Foundation, ο οποίος λειτουργεί και άλλους - πολύγλωσσους και ελεύθερου περιεχομένου ιστοχώρους: - - Wikisource-logo.svg - - Βικιθήκη - Ελεύθερα έγγραφα και πηγές - - Wiktprintable without text.svg - - Βικιλεξικό - Λεξικό και Θησαυρός - - Wikinews-logo.svg - - Βικινέα - Ελεύθερες Ειδήσεις - - Commons-logo.svg - - Commons - Κοινή αποθήκη πολυμέσων - - Wikibooks-logo.svg - - Βικιβιβλία -Εκπαιδευτικά βιβλία και εγχειρίδια ελεύθερης πρόσβασης - - Wikiquote-logo.svg - - Βικιφθέγματα - Συλλογή αποφθεγμάτων - - Wikiversity-logo.svg - - Βικιεπιστήμιο -Ελεύθερο διδακτικό υλικό και δραστηριότητες - - Wikispecies-logo.svg - - Wikispecies - Κατάλογος των ειδών - - Wikimedia Community Logo.svg - - Meta-Wiki - Συντονισμός όλων των προγραμμάτων Wikimedia - - Ανακτήθηκε από - -Προσωπικά εργαλεία - - * Δημιουργία Λογαριασμού/Είσοδος - -Περιοχές ονομάτων - - * Πύλη - * Συζήτηση - -Παραλλαγές - -Εμφανίσεις - - * Ανάγνωση - * Εμφάνιση κώδικα - * Προβολή ιστορικού - -Ενέργειες - -Αναζήτηση - - ____________________ (Submit) Αναζήτηση - -Πλοήγηση - - * Κύρια πύλη - * Κατάλογος άρθρων - * Αξιόλογα άρθρα - * Τρέχοντα γεγονότα - * Τυχαία σελίδα - -Συμμετοχή - - * Πύλη Κοινότητας - * Αγορά - * Πρόσφατες αλλαγές - * Βοήθεια - * Επικοινωνία - * Δωρεές - -Εκτύπωση/εξαγωγή - - * Δημιουργία βιβλίου - * Κατέβασμα ως PDF - * Έκδοση εκτύπωσης - -Εργαλειοθήκη - - * Συνδέσεις προς εδώ - * Σχετικές αλλαγές - * Ειδικές σελίδες - * Σταθερός σύνδεσμος - -Άλλες γλώσσες - - * Τελευταία τροποποίηση 20:32, 8 Σεπτεμβρίου 2011. - * Όλα τα κείμενα είναι διαθέσιμα υπό την Creative Commons - Attribution/Share-Alike License· μπορεί να ισχύουν και πρόσθετοι - όροι. Δείτε τους Όρους Χρήσης για λεπτομέρειες. - - * Πολιτική προσωπικών δεδομένων - * Για τη Βικιπαίδεια - * Αποποίηση ευθυνών - * Προβολή κινητού - diff --git a/src/tests/testdata/el/iso-8859-7.txt b/src/tests/testdata/el/iso-8859-7.txt new file mode 100644 index 0000000..9b036c6 --- /dev/null +++ b/src/tests/testdata/el/iso-8859-7.txt @@ -0,0 +1,3 @@ + ISO 8859-7, , 8- , ISO 8859. . + + 1987 ELOT 928, 1986. 2003, , . diff --git a/src/tests/testdata/el/utf-8.txt b/src/tests/testdata/el/utf-8.txt new file mode 100644 index 0000000..688779f --- /dev/null +++ b/src/tests/testdata/el/utf-8.txt @@ -0,0 +1,3 @@ +Το UTF-8 (8-bit Unicode Transformation Format) είναι ένα μη-απωλεστικό σχήμα κωδικοποίησης χαρακτήρων μεταβλητού μήκους για το πρότυπο Unicode που δημιουργήθηκε από τους Ken Thompson και Rob Pike. Χρησιμοποιεί ομάδες από byte για να αναπαραστήσει τα κωδικά σημεία του Unicode. Είναι ιδιαίτερα χρήσιμο για μετάδοση δεδομένων σε 8bit συστήματα ηλεκτρονικού ταχυδρομείου. + +Συγκεκριμένα χρησιμοποιεί ένα μέχρι τέσσερα byte ανά χαρακτήρα ανάλογα με το σύμβολο και το κωδικό του σημείο. Για παράδειγμα χρειάζεται μόνο ένα byte του UTF-8 για την κωδικοποίηση των 128 ASCII χαρακτήρες στο διάστημα του Unicode U+0000 μέχρι U+007F. diff --git a/src/tests/testdata/el/windows-1253.txt b/src/tests/testdata/el/windows-1253.txt new file mode 100644 index 0000000..3ccac7b --- /dev/null +++ b/src/tests/testdata/el/windows-1253.txt @@ -0,0 +1,5 @@ +Windows-1253 + + Windows-1253. ( Windows-1253) "A2". ( ), , . + + diff --git a/src/tests/testdata/en/UTF-8/wikitop_en_UTF-8.txt b/src/tests/testdata/en/UTF-8/wikitop_en_UTF-8.txt deleted file mode 100644 index 2ba6189..0000000 --- a/src/tests/testdata/en/UTF-8/wikitop_en_UTF-8.txt +++ /dev/null @@ -1,275 +0,0 @@ - Henry Wood (1869–1944) was an English conductor best known for his - association with London's annual series of promenade concerts, known as - the Proms. Wood started his career as an organist. During his studies - at the Royal Academy of Music, he came under the influence of the voice - teacher Manuel Garcia and became his accompanist. After similar work - for Richard D'Oyly Carte's opera companies on the works of Arthur - Sullivan and others, Wood became the conductor of a small operatic - touring company. From the mid-1890s until his death, Wood focused on - concert conducting. He was engaged by the impresario Robert Newman to - conduct a series of promenade concerts at the Queen's Hall, offering a - mixture of classical and popular music at low prices. By the 1920s, - Wood had steered the repertoire entirely to classical music. In - addition to the Proms, he conducted concerts and festivals throughout - the country and also trained the student orchestra at the Royal Academy - of Music. He had an enormous influence on the musical life of Britain - over his long career: he and Newman greatly improved access to - classical music, and Wood raised the standard of orchestral playing and - nurtured the taste of the public, presenting a vast repertoire of music - spanning four centuries. (more...) - - Recently featured: Abdul Karim – Royal National College for the Blind – - Alexandre Banza - Archive – By email – More featured articles... - - Did you know... - - From Wikipedia's newest content: - - Black Kite, known in military slang as the 'shite-hawk' - * ... that, contrary to claims of a medieval origin for the term - shite-hawk, it originated as British military slang for the Black - Kite (pictured)? - * ... that Major League Baseball umpire Paul Sentell collapsed on the - field during a game and died a few days later? - * ... that Captain Thomas Fortescue Kennedy was first-lieutenant of - HMS Temeraire at Trafalgar in 1805, and her last commander at - Sheerness in 1838? - * ... that Bach finished the opening chorus of his cantata Wer sich - selbst erhöhet, der soll erniedriget werden, BWV 47, with a repeat - of the ritornello, adding the complete text of Luke 14:11? - * ... that poet Antonio Machado was born at the Palacio de las Dueñas - in Seville? - * ... that triathletes Chrissie Wellington and Emma Snowsill are - among the World and Olympic champions who have been coached by - Brett Sutton? - * ... that despite leaving school at age 14, Thomas Kirkman became - one of 19th-century England's leading mathematicians and helped - found combinatorial design theory? - Archive – Start a new article – Nominate an article - - - In the news - - Muammar Gaddafi - * Ousted Libyan leader Muammar Gaddafi (pictured) is killed in Sirte, - with National Transitional Council forces taking control of the - city. - * After five years in captivity, Israeli soldier Gilad Shalit is - freed in exchange for the release of 1,027 Palestinian prisoners. - * British writer Julian Barnes wins the Man Booker Prize for his - novel The Sense of an Ending. - * British IndyCar driver Dan Wheldon dies after a crash at the - IndyCar World Championship at Las Vegas Motor Speedway. - * The United States sends 100 military advisers to help defeat the - Lord's Resistance Army in central Africa. - Wikinews – Recent deaths – More current events... - - On this day... - - October 21: Trafalgar Day in various Commonwealth countries - - Solomon R. Guggenheim Museum - * 1096 – The Seljuk forces of Kilij Arslan destroyed the army of the - People's Crusade as it marched toward Nicaea. - * 1520 – The islands of Saint Pierre and Miquelon were discovered by - Portuguese explorer João Álvares Fagundes, who named them "Islands - of the 11,000 Virgins". - * 1805 – Napoleonic Wars: Lord Nelson signalled "England expects that - every man will do his duty" to the rest of his Royal Navy forces - before they defeated Pierre-Charles Villeneuve and his combined - French and Spanish navy at the Battle of Trafalgar off the coast of - Spain's Cape Trafalgar. - * 1959 – The Solomon R. Guggenheim Museum (pictured), designed by - American architect Frank Lloyd Wright, opened in New York City. - * 1978 – After reporting contact with an unidentified aircraft, - Frederick Valentich disappeared in unexplained circumstances while - piloting a Cessna 182L light aircraft over the Bass Strait to King - Island, Australia. - - More anniversaries: October 20 – October 21 – October 22 - Archive – By email – List of historical anniversaries - It is now October 21, 2011 (UTC) – Refresh this page - - Today's featured picture - - - Mount Tarawera rift crater - - Crumbling scoria cliffs surround the rift at the summit of Mount - Tarawera, a volcano near Rotorua in New Zealand's North Island. This - 500 m (1,600 ft) wide rift was created in 1886 in the country's largest - historical eruption. The famed Pink and White Terraces were lost during - the eruption, and over one hundred people were killed. - Photo: Avenue - - Recently featured: Big wave surfing – Heathland in Tasmania – - Plectroctena sp. ants - Archive – More featured pictures... - -Other areas of Wikipedia - - * Community portal – Bulletin board, projects, resources and - activities covering a wide range of Wikipedia areas. - * Help desk – Ask questions about using Wikipedia. - * Local embassy – For Wikipedia-related communication in languages - other than English. - * Reference desk – Serving as virtual librarians, Wikipedia - volunteers tackle your questions on a wide range of subjects. - * Site news – Announcements, updates, articles and press releases on - Wikipedia and the Wikimedia Foundation. - * Village pump – For discussions about Wikipedia itself, including - areas for technical issues and policies. - -Wikipedia's sister projects - - Wikipedia is hosted by the Wikimedia Foundation, a non-profit - organization that also hosts a range of other projects: - Commons Commons - Free media repository Wikiquote Wikiquote - Collection of quotations Wikiversity Wikiversity - Free learning materials and activities - Wikibooks Wikibooks - Free textbooks and manuals Wikisource Wikisource - Free-content library Wiktionary Wiktionary - Dictionary and thesaurus - Wikinews Wikinews - Free-content news Wikispecies Wikispecies - Directory of species Meta-Wiki Meta-Wiki - Wikimedia project coordination - -Wikipedia languages - - This Wikipedia is written in English. Started in 2001 (2001), it - currently contains 3,772,382 articles. Many other Wikipedias are - available; some of the largest are listed below. - * More than 650,000 articles: Deutsch · Español · Français · - Italiano · Nederlands · 日本語 · Polski · Português · Русский - * More than 150,000 articles: العربية · Bahasa Indonesia · Català · - Česky · Dansk · Esperanto · فارسی · 한국어 · Magyar · ‪Norsk - (bokmål)‬ · Română · Suomi · Svenska · Tiếng Việt · Türkçe · - Українська · 中文 - * More than 50,000 articles: Bahasa Melayu · Български · Eesti · - Ελληνικά · Simple English · Euskara · Galego · עברית · Hrvatski · - Lietuvių · ‪Norsk (nynorsk)‬ · Slovenčina · Slovenščina · Српски / - Srpski · ไทย - - Complete list of Wikipedias - Retrieved from - "http://en.wikipedia.org/w/index.php?title=Main_Page&oldid=447996010" - Hidden categories: - * Article Feedback Blacklist - -Personal tools - - * Log in / create account - -Namespaces - - * Main Page - * Discussion - -Variants - -Views - - * Read - * View source - * View history - -Actions - -Search - - ____________________ (Submit) Search - -Navigation - - * Main page - * Contents - * Featured content - * Current events - * Random article - * Donate to Wikipedia - -Interaction - - * Help - * About Wikipedia - * Community portal - * Recent changes - * Contact Wikipedia - -Toolbox - - * What links here - * Related changes - * Upload file - * Special pages - * Permanent link - * Cite this page - -Print/export - - * Create a book - * Download as PDF - * Printable version - -Languages - - * Simple English - * العربية - * Bahasa Indonesia - * Bahasa Melayu - * Български - * Català - * Česky - * Dansk - * Deutsch - * Eesti - * Ελληνικά - * Español - * Esperanto - * Euskara - * فارسی - * Français - * Galego - * 한국어 - * עברית - * Hrvatski - * Italiano - * Lietuvių - * Magyar - * Nederlands - * 日本語 - * ‪Norsk (bokmål)‬ - * ‪Norsk (nynorsk)‬ - * Polski - * Português - * Română - * Русский - * Slovenčina - * Slovenščina - * Српски / Srpski - * Suomi - * Svenska - * ไทย - * Tiếng Việt - * Türkçe - * Українська - * 中文 - - * This page was last modified on 2 September 2011 at 06:18. - * Text is available under the Creative Commons Attribution-ShareAlike - License; additional terms may apply. See Terms of use for details. - Wikipedia® is a registered trademark of the Wikimedia Foundation, - Inc., a non-profit organization. - * Contact us - - * Privacy policy - * About Wikipedia - * Disclaimers - * Mobile view - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/en/WINDOWS-1252/wikitop_en_WINDOWS-1252.txt b/src/tests/testdata/en/WINDOWS-1252/wikitop_en_WINDOWS-1252.txt deleted file mode 100644 index da9038e..0000000 --- a/src/tests/testdata/en/WINDOWS-1252/wikitop_en_WINDOWS-1252.txt +++ /dev/null @@ -1,275 +0,0 @@ - Henry Wood (18691944) was an English conductor best known for his - association with London's annual series of promenade concerts, known as - the Proms. Wood started his career as an organist. During his studies - at the Royal Academy of Music, he came under the influence of the voice - teacher Manuel Garcia and became his accompanist. After similar work - for Richard D'Oyly Carte's opera companies on the works of Arthur - Sullivan and others, Wood became the conductor of a small operatic - touring company. From the mid-1890s until his death, Wood focused on - concert conducting. He was engaged by the impresario Robert Newman to - conduct a series of promenade concerts at the Queen's Hall, offering a - mixture of classical and popular music at low prices. By the 1920s, - Wood had steered the repertoire entirely to classical music. In - addition to the Proms, he conducted concerts and festivals throughout - the country and also trained the student orchestra at the Royal Academy - of Music. He had an enormous influence on the musical life of Britain - over his long career: he and Newman greatly improved access to - classical music, and Wood raised the standard of orchestral playing and - nurtured the taste of the public, presenting a vast repertoire of music - spanning four centuries. (more...) - - Recently featured: Abdul Karim Royal National College for the Blind - Alexandre Banza - Archive By email More featured articles... - - Did you know... - - From Wikipedia's newest content: - - Black Kite, known in military slang as the 'shite-hawk' - * ... that, contrary to claims of a medieval origin for the term - shite-hawk, it originated as British military slang for the Black - Kite (pictured)? - * ... that Major League Baseball umpire Paul Sentell collapsed on the - field during a game and died a few days later? - * ... that Captain Thomas Fortescue Kennedy was first-lieutenant of - HMS Temeraire at Trafalgar in 1805, and her last commander at - Sheerness in 1838? - * ... that Bach finished the opening chorus of his cantata Wer sich - selbst erhhet, der soll erniedriget werden, BWV 47, with a repeat - of the ritornello, adding the complete text of Luke 14:11? - * ... that poet Antonio Machado was born at the Palacio de las Dueas - in Seville? - * ... that triathletes Chrissie Wellington and Emma Snowsill are - among the World and Olympic champions who have been coached by - Brett Sutton? - * ... that despite leaving school at age 14, Thomas Kirkman became - one of 19th-century England's leading mathematicians and helped - found combinatorial design theory? - Archive Start a new article Nominate an article - - - In the news - - Muammar Gaddafi - * Ousted Libyan leader Muammar Gaddafi (pictured) is killed in Sirte, - with National Transitional Council forces taking control of the - city. - * After five years in captivity, Israeli soldier Gilad Shalit is - freed in exchange for the release of 1,027 Palestinian prisoners. - * British writer Julian Barnes wins the Man Booker Prize for his - novel The Sense of an Ending. - * British IndyCar driver Dan Wheldon dies after a crash at the - IndyCar World Championship at Las Vegas Motor Speedway. - * The United States sends 100 military advisers to help defeat the - Lord's Resistance Army in central Africa. - Wikinews Recent deaths More current events... - - On this day... - - October 21: Trafalgar Day in various Commonwealth countries - - Solomon R. Guggenheim Museum - * 1096 The Seljuk forces of Kilij Arslan destroyed the army of the - People's Crusade as it marched toward Nicaea. - * 1520 The islands of Saint Pierre and Miquelon were discovered by - Portuguese explorer Joo lvares Fagundes, who named them "Islands - of the 11,000 Virgins". - * 1805 Napoleonic Wars: Lord Nelson signalled "England expects that - every man will do his duty" to the rest of his Royal Navy forces - before they defeated Pierre-Charles Villeneuve and his combined - French and Spanish navy at the Battle of Trafalgar off the coast of - Spain's Cape Trafalgar. - * 1959 The Solomon R. Guggenheim Museum (pictured), designed by - American architect Frank Lloyd Wright, opened in New York City. - * 1978 After reporting contact with an unidentified aircraft, - Frederick Valentich disappeared in unexplained circumstances while - piloting a Cessna 182L light aircraft over the Bass Strait to King - Island, Australia. - - More anniversaries: October 20 October 21 October 22 - Archive By email List of historical anniversaries - It is now October 21, 2011 (UTC) Refresh this page - - Today's featured picture - - - Mount Tarawera rift crater - - Crumbling scoria cliffs surround the rift at the summit of Mount - Tarawera, a volcano near Rotorua in New Zealand's North Island. This - 500 m (1,600 ft) wide rift was created in 1886 in the country's largest - historical eruption. The famed Pink and White Terraces were lost during - the eruption, and over one hundred people were killed. - Photo: Avenue - - Recently featured: Big wave surfing Heathland in Tasmania - Plectroctena sp. ants - Archive More featured pictures... - -Other areas of Wikipedia - - * Community portal Bulletin board, projects, resources and - activities covering a wide range of Wikipedia areas. - * Help desk Ask questions about using Wikipedia. - * Local embassy For Wikipedia-related communication in languages - other than English. - * Reference desk Serving as virtual librarians, Wikipedia - volunteers tackle your questions on a wide range of subjects. - * Site news Announcements, updates, articles and press releases on - Wikipedia and the Wikimedia Foundation. - * Village pump For discussions about Wikipedia itself, including - areas for technical issues and policies. - -Wikipedia's sister projects - - Wikipedia is hosted by the Wikimedia Foundation, a non-profit - organization that also hosts a range of other projects: - Commons Commons - Free media repository Wikiquote Wikiquote - Collection of quotations Wikiversity Wikiversity - Free learning materials and activities - Wikibooks Wikibooks - Free textbooks and manuals Wikisource Wikisource - Free-content library Wiktionary Wiktionary - Dictionary and thesaurus - Wikinews Wikinews - Free-content news Wikispecies Wikispecies - Directory of species Meta-Wiki Meta-Wiki - Wikimedia project coordination - -Wikipedia languages - - This Wikipedia is written in English. Started in 2001 (2001), it - currently contains 3,772,382 articles. Many other Wikipedias are - available; some of the largest are listed below. - * More than 650,000 articles: Deutsch Espaol Franais - Italiano Nederlands Polski Portugus - * More than 150,000 articles: Bahasa Indonesia Catal - esky Dansk Esperanto Magyar Norsk - (bokml) Romn Suomi Svenska Ting Vit Trke - - * More than 50,000 articles: Bahasa Melayu Eesti - Simple English Euskara Galego Hrvatski - Lietuvi Norsk (nynorsk) Slovenina Slovenina / - Srpski - - Complete list of Wikipedias - Retrieved from - "http://en.wikipedia.org/w/index.php?title=Main_Page&oldid=447996010" - Hidden categories: - * Article Feedback Blacklist - -Personal tools - - * Log in / create account - -Namespaces - - * Main Page - * Discussion - -Variants - -Views - - * Read - * View source - * View history - -Actions - -Search - - ____________________ (Submit) Search - -Navigation - - * Main page - * Contents - * Featured content - * Current events - * Random article - * Donate to Wikipedia - -Interaction - - * Help - * About Wikipedia - * Community portal - * Recent changes - * Contact Wikipedia - -Toolbox - - * What links here - * Related changes - * Upload file - * Special pages - * Permanent link - * Cite this page - -Print/export - - * Create a book - * Download as PDF - * Printable version - -Languages - - * Simple English - * - * Bahasa Indonesia - * Bahasa Melayu - * - * Catal - * esky - * Dansk - * Deutsch - * Eesti - * - * Espaol - * Esperanto - * Euskara - * - * Franais - * Galego - * - * - * Hrvatski - * Italiano - * Lietuvi - * Magyar - * Nederlands - * - * Norsk (bokml) - * Norsk (nynorsk) - * Polski - * Portugus - * Romn - * - * Slovenina - * Slovenina - * / Srpski - * Suomi - * Svenska - * - * Ting Vit - * Trke - * - * - - * This page was last modified on 2 September 2011 at 06:18. - * Text is available under the Creative Commons Attribution-ShareAlike - License; additional terms may apply. See Terms of use for details. - Wikipedia is a registered trademark of the Wikimedia Foundation, - Inc., a non-profit organization. - * Contact us - - * Privacy policy - * About Wikipedia - * Disclaimers - * Mobile view - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/en/ascii.txt b/src/tests/testdata/en/ascii.txt new file mode 100644 index 0000000..8ffb485 --- /dev/null +++ b/src/tests/testdata/en/ascii.txt @@ -0,0 +1,4 @@ +This is an ASCII TEST. +We still want uchardet to detect it as ASCII, even with the presence of +an escape character:  +Or with the HZ encoding escape sequence: ~{ diff --git a/src/tests/testdata/eo/iso-8859-3.txt b/src/tests/testdata/eo/iso-8859-3.txt new file mode 100644 index 0000000..9f35d8a --- /dev/null +++ b/src/tests/testdata/eo/iso-8859-3.txt @@ -0,0 +1,7 @@ +Esperanto (origine Lingvo Internacia) estas la plej disvastigita internacia +planlingvo.[3] La nomo venas de la kanomo "Dr-o Esperanto", sub kiu la juda +kuracisto Ludoviko Lazaro Zamenhofo en la jaro 1887 publikigis la bazon de la +lingvo. La unua versio, la rusa, ricevis la cenzuran permeson disvastii en la +26-a de julio; i tiun daton oni konsideras la naskitago de Esperanto[4][5]. Li +intencis krei facile lerneblan netralan lingvon, tagan por uzo en la +internacia komunikado, tamen ne anstataigi aliajn, naciajn lingvojn. diff --git a/src/tests/testdata/es/UTF-8/wikitop_es_UTF-8.txt b/src/tests/testdata/es/UTF-8/wikitop_es_UTF-8.txt deleted file mode 100644 index e50ad7f..0000000 --- a/src/tests/testdata/es/UTF-8/wikitop_es_UTF-8.txt +++ /dev/null @@ -1,320 +0,0 @@ -Wikipedia:Portada - - De Wikipedia, la enciclopedia libre - Saltar a: navegación, búsqueda - - - Bienvenidos a Wikipedia, - la enciclopedia de contenido libre que todos pueden editar. - Acceso seguro · Acceso WAP (alternativo) · Contacto · Donaciones · - Libro de visitas - - 15:18 UTC – viernes, - 21 de octubre de 2011. - 837.132 artículos en español. - - Participación y comunidad - - ¿Cómo colaborar? · Bienvenida · Primeros pasos - Tutorial · Contenidos de ayuda - Café · Preguntas frecuentes · Los cinco pilares - - Búsquedas y consultas - - Índice de categorías · Todas las categorías - Índice alfabético · Todos los artículos - Portales temáticos · Explorar Wikipedia - - Artículo destacado Artículo destacado - Radiohead - - In Rainbows (en español, «en arcoíris») es el séptimo álbum de estudio - de la banda inglesa de rock alternativo Radiohead. Originalmente se - lanzó el 10 de octubre de 2007 en formato digital y se convino que los - interesados en descargarlo pagaran lo que creyeran conveniente. En - segundo término, se lanzó en formato CD en muchos países durante la - última semana de ese año. En Estados Unidos, se puso a la venta el 1 de - enero de 2008. In Rainbows fue la primera producción tras la - finalización de su contrato con EMI, rematando también la brecha más - grande entre sus álbumes de estudio. - - Radiohead trabajó en el álbum durante más de dos años con los - productores Mark Stent y Nigel Godrich, comenzando a principios de - 2005. Durante el proceso, la banda salió de gira tres meses por Europa - y Estados Unidos a mediados de 2006. Las letras del disco fueron más - personales que en otros trabajos de la banda. El vocalista Thom Yorke - describió varios temas como sus propias versiones de «canciones de - seducción». Radiohead incorporó una amplia variedad de estilos - musicales e instrumentos en el álbum, no haciendo uso exclusivo de - música electrónica y arreglos para instrumentos de cuerda, sino también - de piano, celesta y ondas Martenot. - Leer más... - Anteriores: Basílica de San Pedro, Elizabeth Bowes-Lyon, Brad Pitt. - Artículo bueno Artículo bueno - Testudo hermanni hermanni Mallorca 02.jpg - La tortuga mediterránea (Testudo hermanni) es una de las ocho especies - de tortuga clasificadas tradicionalmente dentro el género Testudo, - junto con especies próximas como por ejemplo la tortuga marginada (T. - marginata), la tortuga mora (T. graeca) o la tortuga rusa - (T.horsfieldii). - - Se trata de reptiles herbívoros y diurnos que pueden alcanzar edades - comparables a las de los humanos, y que se distribuyen desde Cataluña, - al oeste, hasta el límite sur del mar Negro, al este. Junto a la - tortuga marginada (T. marginata) es la única tortuga terrestre de - distribución íntegramente europea. - Leer más... - Anteriores: Batalla de Gitión, Orden de Malta, Lieja - Recurso del día Recurso del día - - Bloque de hielo en una playa cerca de Jökulsárlón (Islandia) El hielo - es agua sólida cristalizada, congelada. Es uno de los tres estados - naturales del agua. Los otros dos estados son el estado líquido y el - estado gaseoso (a 100 °C y a nivel del mar, el vapor). - Archivo - -Portales - - Ciencias naturales y formales - - Ciencias naturales y formales - Astronomía – Biología – Botánica – Física – Matemática – Química - Ciencias humanas y sociales - - Ciencias humanas y sociales - Comunicación – Derecho – Economía – Egiptología – Filosofía – Historia - – Lingüística – Mitología – Política – Psicología – Sociología - Artes - - Artes - Anime y manga – Arquitectura – Cine – Danza – Historieta – Literatura – - Música – Pintura – Teatro – Televisión - Sociedad - - Sociedad - Deporte – Gastronomía – LGBT – Religión – Videojuegos - Tecnologías - - Tecnologías - Biotecnología – Exploración espacial – Informática – Ingeniería – - Medicina – Tecnología - Geografía - - Geografía - África – América – Asia – Europa – Oceanía - - Actualidad Actualidad - Muamar el Gadafi en 2009 - * Erupción de El Hierro de 2011 - * Erupción del complejo volcánico Puyehue-Cordón Caulle de 2011 - * 23 de octubre: Elecciones presidenciales de Argentina de 2011 - * 22-29 de octubre: Semana Internacional de Cine de Valladolid 2011 - * 20 de octubre: Anuncio de un cese definitivo de la actividad armada - por parte de la banda terrorista Euskadi Ta Askatasuna - * 20 de octubre: Muerte de Muamar el Gadafi (en la imagen) - * 19-21 de octubre: Conferencia Mundial sobre los Determinantes - Sociales de la Salud - * 14-30 de octubre: Juegos Panamericanos de 2011 - * 9 de septiembre-23 de octubre: Copa Mundial de Rugby de 2011 - - Fallecimientos - - El militar libio Moatassem Gadafi en 2009 - * 20 de octubre: Muamar el Gadafi, líder libio (69) - * 20 de octubre: Moatassem Gadafi, militar libio (33-34; en la - imagen) - * 19 de octubre: Edison Chará, futbolista colombiano (31) - * 19 de octubre: Nena Jiménez, humorista colombiana (82) - * 18 de octubre: Bernardo Valencia, torero venezolano (59) - * 17 de octubre: Manfred Gerlach, político alemán (83) - * 17 de octubre: Osvaldo Guidi, actor y director argentino (47) - - Conmemoraciones y fiestas - - * 22 de octubre: Día Internacional de la Tartamudez - * 20 de octubre: Día Mundial de la Estadística - - Efemérides: 22 de octubre, 21 de octubre, 20 de octubre - - Véase también: Categoría:Actualidad, 2011, Categoría:2011 - -Efemérides - - Alfred Nobel - * 1805 - La Batalla de Trafalgar supuso la derrota de la escuadra - aliada franco-española frente a la armada británica. - - * 1833 - Nace Alfred Nobel, inventor y químico sueco, famoso por la - dinamita y por los premios que llevan su nombre. - - * 2003 - Muere Arturo Warman, antropólogo, intelectual y político - mexicano. - -Otros proyectos - - Wikipedia es uno de los múltiples proyectos de la Fundación Wikimedia, - una organización sin ánimo de lucro. - Wikimedia Commons Commons - Imágenes y Multimedia Wikcionario Wikcionario - Diccionario libre - Wikisource Wikisource - La biblioteca libre Wikinoticias Wikinoticias - Noticias libres - Wikilibros Wikilibros - Libros de texto y manuales Wikiquote Wikiquote - Colección de citas - Wikiespecies Wikiespecies - Directorio de especies Meta-Wiki Meta-Wiki - Apoyo a los proyectos - Wikiversidad Wikiversidad - Recursos de aprendizaje - -Ediciones de Wikipedia en otros idiomas - - Más de 500 000 artículos: Deutsch (alemán) • English (inglés) • - Français (francés) • Italiano • 日本語 (japonés) • Nederlands (neerlandés) - • Polski (polaco) • Português (portugués) • Русский (ruso) - - Más de 200 000 artículos: Català (catalán) • Česky (checo) • 中文 (chino) - • Suomi (finlandés) • Bokmål (noruego bokmål) • Svenska (sueco) • Tiếng - Việt (vietnamita) • Українська (ucraniano) - Más de 100 000 artículos: العربية (árabe) • Български (búlgaro) • 한국어 - (coreano) • Hrvatski (croata) • Dansk (danés) • Esperanto • Euskara - (euskera) • Slovenčina (eslovaco) • Slovenščina (esloveno) • עברית - (hebreo) • Magyar (húngaro) • Bahasa Indonesia (indonesio) • Lietuvių - (lituano) • Bahasa Melayu (malayo) • فارسی (persa)• Română (rumano) • - Winaray (samareño) • Српски (serbio) • Türkçe (turco) • Volapük - Lista completa — Coordinación plurilingüe - Obtenido de - «http://es.wikipedia.org/w/index.php?title=Wikipedia:Portada&oldid=5022 - 0931» - -Herramientas personales - - * Iniciar sesión / crear cuenta - -Espacios de nombres - - * Portada - * Discusión - -Variantes - -Vistas - - * Leer - * Ver fuente - * Ver historial - -Acciones - -Buscar - - ____________________ (Submit) Buscar - -Navegación - - * Portada - * Portal de la comunidad - * Actualidad - * Cambios recientes - * Páginas nuevas - * Página aleatoria - * Ayuda - * Donaciones - * Notificar un error - -Imprimir/exportar - - * Crear un libro - * Descargar como PDF - * Versión para imprimir - -Herramientas - - * Lo que enlaza aquí - * Cambios en enlazadas - * Subir archivo - * Páginas especiales - * Enlace permanente - -En otros idiomas - - * Aragonés - * العربية - * Asturianu - * Aymar aru - * Български - * ইমার ঠার/বিষ্ণুপ্রিয়া মণিপুরী - * Bosanski - * Català - * Chavacano de Zamboanga - * Cebuano - * Chamoru - * Česky - * Dansk - * Deutsch - * Ελληνικά - * English - * Esperanto - * Estremeñu - * Eesti - * Euskara - * فارسی - * Suomi - * Français - * Galego - * Avañe'ẽ - * עברית - * Hrvatski - * Magyar - * Bahasa Indonesia - * Italiano - * 日本語 - * 한국어 - * Ladino - * Latina - * Lumbaart - * Lietuvių - * Bahasa Melayu - * Mirandés - * Nāhuatl - * नेपाल भाषा - * Nederlands - * ‪Norsk (nynorsk)‬ - * ‪Norsk (bokmål)‬ - * Occitan - * Papiamentu - * Polski - * Português - * Runa Simi - * Română - * Русский - * Slovenčina - * Slovenščina - * Српски / Srpski - * Svenska - * తెలుగు - * ไทย - * Tagalog - * Türkçe - * Українська - * Tiếng Việt - * 中文 - - * Esta página fue modificada por última vez el 2 oct 2011, a las - 19:22. - * El texto está disponible bajo la Licencia Creative Commons - Atribución Compartir Igual 3.0; podrían ser aplicables cláusulas - adicionales. Lee los términos de uso para más información. - Wikipedia® es una marca registrada de la Fundación Wikimedia, Inc., - una organización sin ánimo de lucro. - * Contacto - - * Política de privacidad - * Acerca de Wikipedia - * Limitación de responsabilidad - * Versión para móviles - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/es/WINDOWS-1252/wikitop_es_WINDOWS-1252.txt b/src/tests/testdata/es/WINDOWS-1252/wikitop_es_WINDOWS-1252.txt deleted file mode 100644 index 83cc6d5..0000000 --- a/src/tests/testdata/es/WINDOWS-1252/wikitop_es_WINDOWS-1252.txt +++ /dev/null @@ -1,92 +0,0 @@ - In Rainbows (en espaol, en arcoris) es el sptimo lbum de estudio - de la banda inglesa de rock alternativo Radiohead. Originalmente se - lanz el 10 de octubre de 2007 en formato digital y se convino que los - interesados en descargarlo pagaran lo que creyeran conveniente. En - segundo trmino, se lanz en formato CD en muchos pases durante la - ltima semana de ese ao. En Estados Unidos, se puso a la venta el 1 de - enero de 2008. In Rainbows fue la primera produccin tras la - finalizacin de su contrato con EMI, rematando tambin la brecha ms - grande entre sus lbumes de estudio. - - Radiohead trabaj en el lbum durante ms de dos aos con los - productores Mark Stent y Nigel Godrich, comenzando a principios de - 2005. Durante el proceso, la banda sali de gira tres meses por Europa - y Estados Unidos a mediados de 2006. Las letras del disco fueron ms - personales que en otros trabajos de la banda. El vocalista Thom Yorke - describi varios temas como sus propias versiones de canciones de - seduccin. Radiohead incorpor una amplia variedad de estilos - musicales e instrumentos en el lbum, no haciendo uso exclusivo de - msica electrnica y arreglos para instrumentos de cuerda, sino tambin - de piano, celesta y ondas Martenot. - Leer ms... - Anteriores: Baslica de San Pedro, Elizabeth Bowes-Lyon, Brad Pitt. - Artculo bueno Artculo bueno - Testudo hermanni hermanni Mallorca 02.jpg - La tortuga mediterrnea (Testudo hermanni) es una de las ocho especies - de tortuga clasificadas tradicionalmente dentro el gnero Testudo, - junto con especies prximas como por ejemplo la tortuga marginada (T. - marginata), la tortuga mora (T. graeca) o la tortuga rusa - (T.horsfieldii). - - Se trata de reptiles herbvoros y diurnos que pueden alcanzar edades - comparables a las de los humanos, y que se distribuyen desde Catalua, - al oeste, hasta el lmite sur del mar Negro, al este. Junto a la - tortuga marginada (T. marginata) es la nica tortuga terrestre de - distribucin ntegramente europea. - Leer ms... - Anteriores: Batalla de Gitin, Orden de Malta, Lieja - Recurso del da Recurso del da - - Bloque de hielo en una playa cerca de Jkulsrln (Islandia) El hielo - es agua slida cristalizada, congelada. Es uno de los tres estados - naturales del agua. Los otros dos estados son el estado lquido y el - estado gaseoso (a 100 C y a nivel del mar, el vapor). - Archivo - - Actualidad Actualidad - Muamar el Gadafi en 2009 - * Erupcin de El Hierro de 2011 - * Erupcin del complejo volcnico Puyehue-Cordn Caulle de 2011 - * 23 de octubre: Elecciones presidenciales de Argentina de 2011 - * 22-29 de octubre: Semana Internacional de Cine de Valladolid 2011 - * 20 de octubre: Anuncio de un cese definitivo de la actividad armada - por parte de la banda terrorista Euskadi Ta Askatasuna - * 20 de octubre: Muerte de Muamar el Gadafi (en la imagen) - * 19-21 de octubre: Conferencia Mundial sobre los Determinantes - Sociales de la Salud - * 14-30 de octubre: Juegos Panamericanos de 2011 - * 9 de septiembre-23 de octubre: Copa Mundial de Rugby de 2011 - - Fallecimientos - - El militar libio Moatassem Gadafi en 2009 - * 20 de octubre: Muamar el Gadafi, lder libio (69) - * 20 de octubre: Moatassem Gadafi, militar libio (33-34; en la - imagen) - * 19 de octubre: Edison Char, futbolista colombiano (31) - * 19 de octubre: Nena Jimnez, humorista colombiana (82) - * 18 de octubre: Bernardo Valencia, torero venezolano (59) - * 17 de octubre: Manfred Gerlach, poltico alemn (83) - * 17 de octubre: Osvaldo Guidi, actor y director argentino (47) - - Conmemoraciones y fiestas - - * 22 de octubre: Da Internacional de la Tartamudez - * 20 de octubre: Da Mundial de la Estadstica - - Efemrides: 22 de octubre, 21 de octubre, 20 de octubre - - Vase tambin: Categora:Actualidad, 2011, Categora:2011 - -Efemrides - - Alfred Nobel - * 1805 - La Batalla de Trafalgar supuso la derrota de la escuadra - aliada franco-espaola frente a la armada britnica. - - * 1833 - Nace Alfred Nobel, inventor y qumico sueco, famoso por la - dinamita y por los premios que llevan su nombre. - - * 2003 - Muere Arturo Warman, antroplogo, intelectual y poltico - mexicano. - diff --git a/src/tests/testdata/es/iso-8859-1.txt b/src/tests/testdata/es/iso-8859-1.txt new file mode 100644 index 0000000..f910c39 --- /dev/null +++ b/src/tests/testdata/es/iso-8859-1.txt @@ -0,0 +1,5 @@ +El precio medio de la vivienda nueva es de 2212 EUR/m2, segn datos de la Sociedad +de Tasacin a 31 de diciembre de 2012.156 El precio de la vivienda, sin embargo, +vara ostensiblemente en funcin de las comunidades autnomas y las capitales de +provincia, encontrndose la de mayor valor en Catalua (3146 EUR/m), y en +contraposicin las de Extremadura y Murcia (1271 EUR/m) diff --git a/src/tests/testdata/es/iso-8859-15.txt b/src/tests/testdata/es/iso-8859-15.txt new file mode 100644 index 0000000..bc24d88 --- /dev/null +++ b/src/tests/testdata/es/iso-8859-15.txt @@ -0,0 +1,5 @@ +El precio medio de la vivienda nueva es de 2212 /m2, segn datos de la Sociedad +de Tasacin a 31 de diciembre de 2012.156 El precio de la vivienda, sin embargo, +vara ostensiblemente en funcin de las comunidades autnomas y las capitales de +provincia, encontrndose la de mayor valor en Catalua (3146 /m), y en +contraposicin las de Extremadura y Murcia (1271 /m) diff --git a/src/tests/testdata/es/utf-8.txt b/src/tests/testdata/es/utf-8.txt new file mode 100644 index 0000000..78d2fa7 --- /dev/null +++ b/src/tests/testdata/es/utf-8.txt @@ -0,0 +1,5 @@ +El precio medio de la vivienda nueva es de 2212 €/m2, según datos de la Sociedad +de Tasación a 31 de diciembre de 2012.156 El precio de la vivienda, sin embargo, +varía ostensiblemente en función de las comunidades autónomas y las capitales de +provincia, encontrándose la de mayor valor en Cataluña (3146 €/m²), y en +contraposición las de Extremadura y Murcia (1271 €/m²) diff --git a/src/tests/testdata/es/windows-1252.txt b/src/tests/testdata/es/windows-1252.txt new file mode 100644 index 0000000..9bccea0 --- /dev/null +++ b/src/tests/testdata/es/windows-1252.txt @@ -0,0 +1,5 @@ +El precio medio de la vivienda nueva es de 2212 /m2, segn datos de la Sociedad +de Tasacin a 31 de diciembre de 2012.156 El precio de la vivienda, sin embargo, +vara ostensiblemente en funcin de las comunidades autnomas y las capitales de +provincia, encontrndose la de mayor valor en Catalua (3146 /m), y en +contraposicin las de Extremadura y Murcia (1271 /m) diff --git a/src/tests/testdata/et/iso-8859-13.txt b/src/tests/testdata/et/iso-8859-13.txt new file mode 100644 index 0000000..b4c8124 --- /dev/null +++ b/src/tests/testdata/et/iso-8859-13.txt @@ -0,0 +1,6 @@ +Anton Pavlovit Tehhov oli vene nite- ja novellikirjanik ning praktiseeriv arst. + +Tehhov on eelkige tuntud oma novellide poolest. Tema jutustuste tavaliseks +tegevuspaigaks olid vene vikeasulad ja need ksitlesid hingeksildust, raisatud +nne jms. Tuntud on ka tema pshholoogilised nidendid, kus valitseb kurb ja +lootusetu meeleolu. diff --git a/src/tests/testdata/et/iso-8859-15.txt b/src/tests/testdata/et/iso-8859-15.txt new file mode 100644 index 0000000..fc0509a --- /dev/null +++ b/src/tests/testdata/et/iso-8859-15.txt @@ -0,0 +1,6 @@ +Anton Pavlovit Tehhov oli vene nite- ja novellikirjanik ning praktiseeriv arst. + +Tehhov on eelkige tuntud oma novellide poolest. Tema jutustuste tavaliseks +tegevuspaigaks olid vene vikeasulad ja need ksitlesid hingeksildust, raisatud +nne jms. Tuntud on ka tema pshholoogilised nidendid, kus valitseb kurb ja +lootusetu meeleolu. diff --git a/src/tests/testdata/et/iso-8859-4.txt b/src/tests/testdata/et/iso-8859-4.txt new file mode 100644 index 0000000..d5532b0 --- /dev/null +++ b/src/tests/testdata/et/iso-8859-4.txt @@ -0,0 +1,6 @@ +Anton Pavlovit Tehhov oli vene nite- ja novellikirjanik ning praktiseeriv arst. + +Tehhov on eelkige tuntud oma novellide poolest. Tema jutustuste tavaliseks +tegevuspaigaks olid vene vikeasulad ja need ksitlesid hingeksildust, raisatud +nne jms. Tuntud on ka tema pshholoogilised nidendid, kus valitseb kurb ja +lootusetu meeleolu. diff --git a/src/tests/testdata/et/utf-8.txt b/src/tests/testdata/et/utf-8.txt new file mode 100644 index 0000000..d68c9d3 --- /dev/null +++ b/src/tests/testdata/et/utf-8.txt @@ -0,0 +1,6 @@ +Anton Pavlovitš Tšehhov oli vene näite- ja novellikirjanik ning praktiseeriv arst. + +Tšehhov on eelkõige tuntud oma novellide poolest. Tema jutustuste tavaliseks +tegevuspaigaks olid vene väikeasulad ja need käsitlesid hingeüksildust, raisatud +õnne jms. Tuntud on ka tema psühholoogilised näidendid, kus valitseb kurb ja +lootusetu meeleolu. diff --git a/src/tests/testdata/et/windows-1252.txt b/src/tests/testdata/et/windows-1252.txt new file mode 100644 index 0000000..597e28e --- /dev/null +++ b/src/tests/testdata/et/windows-1252.txt @@ -0,0 +1,6 @@ +Anton Pavlovit Tehhov oli vene nite- ja novellikirjanik ning praktiseeriv arst. + +Tehhov on eelkige tuntud oma novellide poolest. Tema jutustuste tavaliseks +tegevuspaigaks olid vene vikeasulad ja need ksitlesid hingeksildust, raisatud +nne jms. Tuntud on ka tema pshholoogilised nidendid, kus valitseb kurb ja +lootusetu meeleolu. diff --git a/src/tests/testdata/et/windows-1257.txt b/src/tests/testdata/et/windows-1257.txt new file mode 100644 index 0000000..64d3327 --- /dev/null +++ b/src/tests/testdata/et/windows-1257.txt @@ -0,0 +1,6 @@ +Anton Pavlovit Tehhov oli vene nite- ja novellikirjanik ning praktiseeriv arst. + +Tehhov on eelkige tuntud oma novellide poolest. Tema jutustuste tavaliseks +tegevuspaigaks olid vene vikeasulad ja need ksitlesid hingeksildust, raisatud +nne jms. Tuntud on ka tema pshholoogilised nidendid, kus valitseb kurb ja +lootusetu meeleolu. diff --git a/src/tests/testdata/fi/UTF-8/wikitop_fi_UTF-8.txt b/src/tests/testdata/fi/UTF-8/wikitop_fi_UTF-8.txt deleted file mode 100644 index 1ed2ea1..0000000 --- a/src/tests/testdata/fi/UTF-8/wikitop_fi_UTF-8.txt +++ /dev/null @@ -1,271 +0,0 @@ -Wikipedia:Etusivu - - Wikipedia - Loikkaa: valikkoon, hakuun - - Tervetuloa Wikipediaan, vapaaseen tietosanakirjaan. - - Suomenkielisessä Wikipediassa on tällä hetkellä 280 731 artikkelia. - - Haku Wikipediasta Haku • Ajankohtaista • Selaa luokittain • Luo - artikkeli - - Suositellut artikkelit • Hyvät artikkelit • Uudet sivut - - Tervetuloa Wikipediaan Tervetuloa Wikipediaan ja tietoja Wikipediasta - - UKK • Artikkelin luominen • Keskustelu • Neuvonta - - - Suositeltu artikkeli - - Bundesarchiv Bild 102-00457, Emanuel Lasker.jpg - - Emanuel Lasker (24. joulukuuta 1868 – 11. tammikuuta 1941) oli - saksalainen shakin suurmestari, matemaatikko ja filosofi. Hän oli - shakin toinen maailmanmestari ja piti titteliä hallussaan - yhtäjaksoisesti 27 vuotta, 1894–1921, mikä on edelleen ennätys. - - Laskerista tuli shakin maailmanmestari voitettuaan Wilhelm Steinitzin - ottelussa vuonna 1894. Hän puolusti mestaruuttaan onnistuneesti viisi - kertaa: kerran Steinitzia vastaan vuonna 1896, Marshallia vastaan 1907, - Tarraschia vastaan 1908 sekä kahdesti vuonna 1910 Schlechteriä ja - Janowskia vastaan. Hän menetti mestaruutensa vuonna 1921 ottelussa - Capablancaa vastaan, mutta jatkoi aktiivista pelaamista vielä yli - vuosikymmenen ajan. Hän julkaisi useita shakkilehtiä ja -kirjoja sekä - työskenteli matematiikan ja filosofian parissa. - - Lisää suositeltuja artikkeleita sekä hyviä artikkeleita - - - Miten voit auttaa - - Wikipedia tarvitsee apuasi! Punainen linkki tarkoittaa, että aiheesta - ei vielä ole artikkelia. Voit auttaa kirjoittamalla puuttuvista - aiheista. - - Zungarin kaanikunta · Karl Gabrielsson · kehrääjät (heimo) · Norfolk - Southern Railway · Miljoona ruusua · Ichthyornis dispar · - Interline-sopimus · laivanrakennus · itämainen kulttuuri · - varhaisnykyaika · Virgilio Hilario · äänteensiirros · preemio · - energian varastointi · venäläistäminen · jukat · tariffi · - taloudellinen determinismi - - Viikon yhteistyöartikkeleina ovat Florida ja väestöpyramidi. - - Lisää artikkelitoiveita · Osallistu Wikiprojekteihin · Paranna sivuja - Arvioi artikkeleita · Laajenna minitynkiä ja tynkiä - - - Äskettäin kuolleita - - Muammar al-Gaddafi 1-1.jpg - - Muammar Gaddafi (1942–2011) oli Libyan diktaattori 1969–2011. Hänellä - ei ollut virallista virkaa tai asemaa, mutta häntä kunnioitettiin - vallankumouksen veljellisenä johtajana ja ohjaajana virallisissa - valtion tiedotuksissa. - - Katso myös: Dan Wheldon · Betty Driver · Abdoulaye Seye · Reijo Tani · - Heinz Bennent · János Herskó · Kim Brown - - Lisää äskettäin kuolleita - - - - Uutisissa - - Muammar al-Gaddafi at the AU summit.jpg Muammar Gaddafi - * Lapista löytyi tieteelle uusi taigavarpuhämähäkki-eläinlaji. - (21.10.) - * Libyan väliaikaishallinnon mukaan Libyan entinen johtaja Muammar - Gaddafi kuoli kiinniotettaessa. (20.10.) - * Ukrainan entinen pääministeri Julija Tymošenko tuomittiin - seitsemäksi vuodeksi vankeuteen valtaoikeuksien väärinkäytöstä. - (11.10.) - * Thomas Sargent ja Christopher A. Sims jakoivat Nobelin - taloustieteen palkinnon. (10.10.) - * Saksalaisesta Sebastian Vettelistä tuli Formula 1 -sarjan nuorin - kaksinkertainen maailmanmestari. (9.10.) - - Wikiuutiset – Lisää uutisia - - - 21. lokakuuta - - AlfredNobel2.jpg Alfred Nobel - * 1805 – Napoleonin sodat: Amiraali Nelsonin johtama brittien - laivasto voitti ranskalaisten ja espanjalaisten laivaston Espanjan - rannikolla Trafalgarin taistelussa. Nelson sai taistelussa - surmansa. - * 1833 – Dynamiitin keksijä ja Nobel-palkinnon perustaja Alfred Nobel - syntyi. - * 1879 – Thomas Edison keksi ensimmäisen käyttökelpoisen hehkulampun. - * 1929 – Yhdysvaltalainen kirjailija Ursula K. Le Guin syntyi. - * 1969 – Yhdysvaltalainen beatnik-kirjailija Jack Kerouac kuoli. - * 1984 – Ranskalainen elokuvaohjaaja François Truffaut kuoli. - - Lisää tapahtumia sekä vuosipäiviä - - - Tiesitkö, että... - - Apollo-12-LOGO.jpg Apollo 12 -lennon logo. - * ...Apollo 12 -kuulennon varamiehistö oli jättänyt miehistön - tehtävälistan väliin Playboy-lehdestä leikatun alastoman naisen - kuvan ja kehotuksen ”tarkata pinnanmuotoja”? - * ...Karjala ja Lappi ovat entisiä kuntia? - * ...Suomessa on Saaren kartano sekä Mietoisissa, Mäntsälässä, - Tammelassa että Ulvilassa, ja kaikki neljä kartanoa kuuluvat - valtakunnallisesti merkittäviin rakennettuihin - kulttuuriympäristöihin? - - Lisää nippelitietoa - -Wikipedian sisarhankkeet - - Wikipediaa ylläpitää voittoa tavoittelematon Wikimedia Foundation, joka - ylläpitää muitakin monikielisiä vapaan sisällön hankkeita: - - Commons-logo.svg Commons - Yhteisiä mediatiedostoja Wikinews-logo.svg Wikiuutiset - Uutispalvelu Wiktionary-logo-en.svg Wikisanakirja - Sanakirjoja - Wikiquote-logo.svg Wikisitaatit - Sitaattikokoelmia Wikibooks-logo.svg Wikikirjasto - Vapaita oppikirjoja ja oppaita Wikisource-logo.svg Wikiaineisto - Vapaiden tekstien arkisto - Wikispecies-logo.svg Wikispecies - Lajien luettelo Wikiversity-logo.svg Wikiopisto - Vapaa oppimisen verkkoyhteisö Wikimedia Community Logo.svg Meta-Wiki - Wikimedian hankkeiden koordinaatio - -Wikipedia muilla kielillä - - Kymmenen suurinta Wikipediaa artikkeleiden määrällä mitaten: - English (englanti) · Deutsch (saksa) · Français (ranska) · Polski - (puola) · Italiano (italia) · 日本語 (japani) · Español (espanja) · - Nederlands (hollanti) · Português (portugali) · Русский (venäjä) - - Täydellinen luettelo · Viikon yhteistyökäännösartikkeli - Haettu osoitteesta - http://fi.wikipedia.org/w/index.php?title=Wikipedia:Etusivu&oldid=10567 - 816 - -Henkilökohtaiset työkalut - - * Kirjaudu sisään tai luo tunnus - -Nimiavaruudet - - * Etusivu - * Keskustelu - -Muuttujat - -Näkymät - - * Lue - * Lähdekoodi - * Näytä historia - -Toiminnot - -Haku - - ____________________ (Submit) Etsi - -Valikko - - * Etusivu - * Tietoja Wikipediasta - * Kaikki sivut - * Satunnainen artikkeli - -Osallistuminen - - * Ohje - * Kahvihuone - * Ajankohtaista - * Tuoreet muutokset - * Lahjoitukset - -Tulosta tai vie - - * Luo kirja - * Lataa PDF-tiedostona - * Tulostettava versio - -Työkalut - - * Tänne viittaavat sivut - * Linkitettyjen sivujen muutokset - * Toimintosivut - * Ikilinkki - -Muilla kielillä - - * العربية - * Armãneashce - * Bahasa Indonesia - * Bahasa Melayu - * Български - * Català - * Cebuano - * Česky - * Dansk - * Deutsch - * Eesti - * Ελληνικά - * English - * Español - * Esperanto - * Euskara - * فارسی - * Français - * Galego - * 한국어 - * हिन्दी - * Hrvatski - * Italiano - * עברית - * ქართული - * Kreyòl ayisyen - * Lietuvių - * Magyar - * Nederlands - * नेपाल भाषा - * 日本語 - * ‪Norsk (bokmål)‬ - * ‪Norsk (nynorsk)‬ - * Polski - * Português - * Română - * Русский - * Simple English - * Slovenčina - * Slovenščina - * Српски / Srpski - * Svenska - * తెలుగు - * ไทย - * Tiếng Việt - * Türkçe - * Українська - * Volapük - * Winaray - * 中文 - - * Sivua on viimeksi muutettu 19. elokuuta 2011 kello 10.04. - * Teksti on saatavilla Creative Commons Attribution/Share-Alike - -lisenssillä; lisäehtoja voi sisältyä. Katso käyttöehdot. - Wikipedia® on Wikimedia Foundationin rekisteröimä tavaramerkki. - Ongelma artikkelissa? - - * Yksityisyydensuoja - * Tietoja Wikipediasta - * Vastuuvapaus - * Mobiilinäkymä - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/fi/WINDOWS-1252/wikitop_fi_WINDOWS-1252.txt b/src/tests/testdata/fi/WINDOWS-1252/wikitop_fi_WINDOWS-1252.txt deleted file mode 100644 index ddeed57..0000000 --- a/src/tests/testdata/fi/WINDOWS-1252/wikitop_fi_WINDOWS-1252.txt +++ /dev/null @@ -1,271 +0,0 @@ -Wikipedia:Etusivu - - Wikipedia - Loikkaa: valikkoon, hakuun - - Tervetuloa Wikipediaan, vapaaseen tietosanakirjaan. - - Suomenkielisess Wikipediassa on tll hetkell 280 731 artikkelia. - - Haku Wikipediasta Haku Ajankohtaista Selaa luokittain Luo - artikkeli - - Suositellut artikkelit Hyvt artikkelit Uudet sivut - - Tervetuloa Wikipediaan Tervetuloa Wikipediaan ja tietoja Wikipediasta - - UKK Artikkelin luominen Keskustelu Neuvonta - - - Suositeltu artikkeli - - Bundesarchiv Bild 102-00457, Emanuel Lasker.jpg - - Emanuel Lasker (24. joulukuuta 1868 11. tammikuuta 1941) oli - saksalainen shakin suurmestari, matemaatikko ja filosofi. Hn oli - shakin toinen maailmanmestari ja piti titteli hallussaan - yhtjaksoisesti 27 vuotta, 18941921, mik on edelleen enntys. - - Laskerista tuli shakin maailmanmestari voitettuaan Wilhelm Steinitzin - ottelussa vuonna 1894. Hn puolusti mestaruuttaan onnistuneesti viisi - kertaa: kerran Steinitzia vastaan vuonna 1896, Marshallia vastaan 1907, - Tarraschia vastaan 1908 sek kahdesti vuonna 1910 Schlechteri ja - Janowskia vastaan. Hn menetti mestaruutensa vuonna 1921 ottelussa - Capablancaa vastaan, mutta jatkoi aktiivista pelaamista viel yli - vuosikymmenen ajan. Hn julkaisi useita shakkilehti ja -kirjoja sek - tyskenteli matematiikan ja filosofian parissa. - - Lis suositeltuja artikkeleita sek hyvi artikkeleita - - - Miten voit auttaa - - Wikipedia tarvitsee apuasi! Punainen linkki tarkoittaa, ett aiheesta - ei viel ole artikkelia. Voit auttaa kirjoittamalla puuttuvista - aiheista. - - Zungarin kaanikunta Karl Gabrielsson kehrjt (heimo) Norfolk - Southern Railway Miljoona ruusua Ichthyornis dispar - Interline-sopimus laivanrakennus itmainen kulttuuri - varhaisnykyaika Virgilio Hilario nteensiirros preemio - energian varastointi venlistminen jukat tariffi - taloudellinen determinismi - - Viikon yhteistyartikkeleina ovat Florida ja vestpyramidi. - - Lis artikkelitoiveita Osallistu Wikiprojekteihin Paranna sivuja - Arvioi artikkeleita Laajenna minitynki ja tynki - - - skettin kuolleita - - Muammar al-Gaddafi 1-1.jpg - - Muammar Gaddafi (19422011) oli Libyan diktaattori 19692011. Hnell - ei ollut virallista virkaa tai asemaa, mutta hnt kunnioitettiin - vallankumouksen veljellisen johtajana ja ohjaajana virallisissa - valtion tiedotuksissa. - - Katso mys: Dan Wheldon Betty Driver Abdoulaye Seye Reijo Tani - Heinz Bennent Jnos Hersk Kim Brown - - Lis skettin kuolleita - - - - Uutisissa - - Muammar al-Gaddafi at the AU summit.jpg Muammar Gaddafi - * Lapista lytyi tieteelle uusi taigavarpuhmhkki-elinlaji. - (21.10.) - * Libyan vliaikaishallinnon mukaan Libyan entinen johtaja Muammar - Gaddafi kuoli kiinniotettaessa. (20.10.) - * Ukrainan entinen pministeri Julija Tymoenko tuomittiin - seitsemksi vuodeksi vankeuteen valtaoikeuksien vrinkytst. - (11.10.) - * Thomas Sargent ja Christopher A. Sims jakoivat Nobelin - taloustieteen palkinnon. (10.10.) - * Saksalaisesta Sebastian Vettelist tuli Formula 1 -sarjan nuorin - kaksinkertainen maailmanmestari. (9.10.) - - Wikiuutiset Lis uutisia - - - 21. lokakuuta - - AlfredNobel2.jpg Alfred Nobel - * 1805 Napoleonin sodat: Amiraali Nelsonin johtama brittien - laivasto voitti ranskalaisten ja espanjalaisten laivaston Espanjan - rannikolla Trafalgarin taistelussa. Nelson sai taistelussa - surmansa. - * 1833 Dynamiitin keksij ja Nobel-palkinnon perustaja Alfred Nobel - syntyi. - * 1879 Thomas Edison keksi ensimmisen kyttkelpoisen hehkulampun. - * 1929 Yhdysvaltalainen kirjailija Ursula K. Le Guin syntyi. - * 1969 Yhdysvaltalainen beatnik-kirjailija Jack Kerouac kuoli. - * 1984 Ranskalainen elokuvaohjaaja Franois Truffaut kuoli. - - Lis tapahtumia sek vuosipivi - - - Tiesitk, ett... - - Apollo-12-LOGO.jpg Apollo 12 -lennon logo. - * ...Apollo 12 -kuulennon varamiehist oli jttnyt miehistn - tehtvlistan vliin Playboy-lehdest leikatun alastoman naisen - kuvan ja kehotuksen tarkata pinnanmuotoja? - * ...Karjala ja Lappi ovat entisi kuntia? - * ...Suomessa on Saaren kartano sek Mietoisissa, Mntslss, - Tammelassa ett Ulvilassa, ja kaikki nelj kartanoa kuuluvat - valtakunnallisesti merkittviin rakennettuihin - kulttuuriympristihin? - - Lis nippelitietoa - -Wikipedian sisarhankkeet - - Wikipediaa yllpit voittoa tavoittelematon Wikimedia Foundation, joka - yllpit muitakin monikielisi vapaan sislln hankkeita: - - Commons-logo.svg Commons - Yhteisi mediatiedostoja Wikinews-logo.svg Wikiuutiset - Uutispalvelu Wiktionary-logo-en.svg Wikisanakirja - Sanakirjoja - Wikiquote-logo.svg Wikisitaatit - Sitaattikokoelmia Wikibooks-logo.svg Wikikirjasto - Vapaita oppikirjoja ja oppaita Wikisource-logo.svg Wikiaineisto - Vapaiden tekstien arkisto - Wikispecies-logo.svg Wikispecies - Lajien luettelo Wikiversity-logo.svg Wikiopisto - Vapaa oppimisen verkkoyhteis Wikimedia Community Logo.svg Meta-Wiki - Wikimedian hankkeiden koordinaatio - -Wikipedia muilla kielill - - Kymmenen suurinta Wikipediaa artikkeleiden mrll mitaten: - English (englanti) Deutsch (saksa) Franais (ranska) Polski - (puola) Italiano (italia) (japani) Espaol (espanja) - Nederlands (hollanti) Portugus (portugali) (venj) - - Tydellinen luettelo Viikon yhteistyknnsartikkeli - Haettu osoitteesta - http://fi.wikipedia.org/w/index.php?title=Wikipedia:Etusivu&oldid=10567 - 816 - -Henkilkohtaiset tykalut - - * Kirjaudu sisn tai luo tunnus - -Nimiavaruudet - - * Etusivu - * Keskustelu - -Muuttujat - -Nkymt - - * Lue - * Lhdekoodi - * Nyt historia - -Toiminnot - -Haku - - ____________________ (Submit) Etsi - -Valikko - - * Etusivu - * Tietoja Wikipediasta - * Kaikki sivut - * Satunnainen artikkeli - -Osallistuminen - - * Ohje - * Kahvihuone - * Ajankohtaista - * Tuoreet muutokset - * Lahjoitukset - -Tulosta tai vie - - * Luo kirja - * Lataa PDF-tiedostona - * Tulostettava versio - -Tykalut - - * Tnne viittaavat sivut - * Linkitettyjen sivujen muutokset - * Toimintosivut - * Ikilinkki - -Muilla kielill - - * - * Armneashce - * Bahasa Indonesia - * Bahasa Melayu - * - * Catal - * Cebuano - * esky - * Dansk - * Deutsch - * Eesti - * - * English - * Espaol - * Esperanto - * Euskara - * - * Franais - * Galego - * - * - * Hrvatski - * Italiano - * - * - * Kreyl ayisyen - * Lietuvi - * Magyar - * Nederlands - * - * - * Norsk (bokml) - * Norsk (nynorsk) - * Polski - * Portugus - * Romn - * - * Simple English - * Slovenina - * Slovenina - * / Srpski - * Svenska - * - * - * Ting Vit - * Trke - * - * Volapk - * Winaray - * - - * Sivua on viimeksi muutettu 19. elokuuta 2011 kello 10.04. - * Teksti on saatavilla Creative Commons Attribution/Share-Alike - -lisenssill; lisehtoja voi sislty. Katso kyttehdot. - Wikipedia on Wikimedia Foundationin rekisterim tavaramerkki. - Ongelma artikkelissa? - - * Yksityisyydensuoja - * Tietoja Wikipediasta - * Vastuuvapaus - * Mobiilinkym - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/fi/iso-8859-1.txt b/src/tests/testdata/fi/iso-8859-1.txt new file mode 100644 index 0000000..3d584ff --- /dev/null +++ b/src/tests/testdata/fi/iso-8859-1.txt @@ -0,0 +1,8 @@ +Termi science fiction on amerikkalaisen tieteislehtien toimittajan Hugo +Gernsbackin keksim. Suomessa termin tieteiskirjallisuus loi tohtori Eino +Kauppinen 1950-luvun alkupuolella. +Tieteiskirjallisuudelle on laadittu erilaisia mritelmi. Tieteiskirjallisuuden +rajat eivt ole yksiselitteisen selket. Tieteiskirjallisuus lhenee monia +kirjallisuudenlajeja, erityisesti kauhu- ja fantasiakirjallisuutta. Nill +kolmella lajilla onkin yhteiset juuret 1800-lukua edeltvss ei-realistisessa +kirjallisuudessa. diff --git a/src/tests/testdata/fi/utf-8.txt b/src/tests/testdata/fi/utf-8.txt new file mode 100644 index 0000000..ca98172 --- /dev/null +++ b/src/tests/testdata/fi/utf-8.txt @@ -0,0 +1,8 @@ +Termi science fiction on amerikkalaisen tieteislehtien toimittajan Hugo +Gernsbackin keksimä. Suomessa termin tieteiskirjallisuus loi tohtori Eino +Kauppinen 1950-luvun alkupuolella. +Tieteiskirjallisuudelle on laadittu erilaisia määritelmiä. Tieteiskirjallisuuden +rajat eivät ole yksiselitteisen selkeät. Tieteiskirjallisuus lähenee monia +kirjallisuudenlajeja, erityisesti kauhu- ja fantasiakirjallisuutta. Näillä +kolmella lajilla onkin yhteiset juuret 1800-lukua edeltävässä ei-realistisessa +kirjallisuudessa. diff --git a/src/tests/testdata/fr/UTF-8/wikitop_fr_UTF-8.txt b/src/tests/testdata/fr/UTF-8/wikitop_fr_UTF-8.txt deleted file mode 100644 index 744af0d..0000000 --- a/src/tests/testdata/fr/UTF-8/wikitop_fr_UTF-8.txt +++ /dev/null @@ -1,292 +0,0 @@ -Wikipédia:Accueil principal - - Une page de Wikipédia, l'encyclopédie libre. - Aller à : Navigation, rechercher - - Bienvenue sur Wikipédia - Le projet d’encyclopédie libre que vous pouvez améliorer 1 163 247 - articles en français - Version pour appareil mobile - - Culture — Géographie — Histoire — Sciences — Société — Technologies - Liste des portails thématiques - - Lumière sur - - L’histoire des Juifs en France, l’une des plus anciennes communautés - juives d’Europe occidentale, semble remonter au I^er siècle de l’ère - commune et se poursuit jusqu’à nos jours. - - Arrivés en Gaule peu après sa conquête par l’Empire romain, les Juifs - s’y maintiennent sous les Mérovingiens, et connaissent une période de - prospérité sous les Carolingiens. Au XI^e siècle, la France devient un - pôle florissant de la culture juive, abritant, au nord, les communautés - ashkénazes, parmi lesquelles fleurit l’école de Rachi et de ses - continuateurs, et, au sud, les Juifs de Provence et du Languedoc. La - situation se détériore fortement après les Croisades, auxquelles font - suite les procès du Talmud, et les expulsions, temporaires puis - définitives. Un millénaire après leur établissement, il ne reste plus - de Juifs que dans les états papaux. - - Environ un siècle après l’expulsion des Juifs de la péninsule ibérique, - des crypto-Juifs originaires du Portugal s’installent à Bordeaux et - Bayonne. Au XVII^e siècle, les Juifs d’Alsace et de Lorraine se - retrouvent eux aussi sous la juridiction de la France, suite aux - traités de Westphalie. - - Les Juifs de France sont les premiers à jouir de l’émancipation que la - France leur accorde au début de la Révolution française, tant dans la - métropole que dans les colonies. Cependant, au « franco-judaïsme » - s’inscrivant dans le cadre de la laïcité en France, répond un « - antisémitisme à la française », qui s’exacerbe notamment lors de - l’affaire Dreyfus, et sous le régime de Vichy. Brutalement isolés du - reste de la population et poursuivis avec un zèle particulier par la - Milice, 75 000 Juifs meurent au cours de l’Occupation, parmi lesquels - de nombreux réfugiés d’Europe de l’Est ou d’Allemagne. - - La France demeure cependant le choix naturel pour nombre de Juifs - contraints de quitter l’Égypte et l’Afrique du Nord dans les années - 1950 et 1960. La communauté juive de France, jusqu’alors - essentiellement ashkénaze et assimilée, devient majoritairement - séfarade et attachée aux traditions. Elle est, de nos jours, la plus - importante d’Europe, et comprend 483 500 personnes, qui habitent - principalement Paris, Marseille, Lyon, Nice, Toulouse et Strasbourg. - Toutes les tendances « religieuses » s’y rencontrent, depuis les Juifs - ultra-orthodoxes jusqu’aux Juifs assimilés. - * Lire la suite - - Contenus de qualité • Bons contenus • Sélection • Programme - - Actualités et évènements - - Évènements en cours : Crise politique belge · Printemps arabe · Crise - de la dette dans la zone euro · Crise alimentaire dans la Corne de - l’Afrique · Mouvement Occupy Wall Street · Coupe du monde de rugby à - XV · Jeux panaméricains - __________________________________________________________________ - - * 21 octobre : L'organisation terroriste basque ETA annonce l'arrêt - définitif de la lutte armée, après plus de quarante ans d'actions - terroristes ayant causé plus de huit cents morts. - - Muammar al-Gaddafi at the AU summit.jpg - * 20 octobre : le CNT annonce la mort de Mouammar Kadhafi (photo) - lors de la prise de Syrte, dernière ville sous son contrôle. - * 18 octobre : après cinq années de captivité, le soldat israélien - Gilad Shalit est libéré en échange de la libération de - 1 027 prisonniers palestiniens. - * 13 octobre : mariage de Jigme Khesar Namgyel Wangchuck, roi du - Bhoutan, et Jetsun Pema. - * 11 octobre : élection présidentielle au Liberia. - * 11 octobre : l’ex-Premier ministre et actuelle chef de l’opposition - en Ukraine Ioulia Tymochenko est condamnée à sept ans de prison - pour abus de pouvoir. - - Octobre 2011 • Éphéméride du jour • Wikinews • Modifier - - Le saviez-vous ? - - vue en coupe d'un REP - * Les deux tiers des 440 réacteurs nucléaires qui produisent de - l'électricité dans le monde sont des réacteurs à eau pressurisée - (REP). - * Le Britannique Fauja Singh détient neuf records du monde - d’athlétisme pour les plus de cent ans, dont celui du marathon. - * Vakif, dans la province de Hatay, est le dernier village arménien - de Turquie. - * Depuis le 27 mars 1901, l’île des Faisans, sur la Bidassoa, change - de souveraineté tous les six mois entre la France et l’Espagne. - * Entre 1990 et 2000, la déforestation a fait perdre à la forêt - amazonienne une superficie comparable à celle de la France. - - Archives • Modifier - - Image du jour - - - Le Quintette de Stephan, du nom de l'astronome Édouard Stephan, est un - groupement visuel de galaxies situé dans la constellation de Pégase. - (définition réelle 6 064 × 6 760) - - Le Quintette de Stephan, du nom de l'astronome Édouard Stephan, est un - groupement visuel de galaxies situé dans la constellation de Pégase. - (définition réelle 6 064 × 6 760 – obtenir en poster) - - Archives • Voir les images - Images de qualité sur Wikimédia Commons - - - Présentation - - Wikipédia est un projet d’encyclopédie collective établie sur Internet, - universelle, multilingue et fonctionnant sur le principe du wiki. - Wikipédia a pour objectif d’offrir un contenu librement réutilisable, - objectif et vérifiable, que chacun peut modifier et améliorer. - - Le cadre du projet est défini par des principes fondateurs. Son contenu - est sous licence Creative Commons by-sa et peut être copié et réutilisé - sous la même licence – même à des fins commerciales – sous réserve d'en - respecter les conditions. - - Actuellement, Wikipédia en français compte plus de deux mille articles - distingués comme « articles de qualité » ou comme « bons articles ». - À propos de Wikipédia • Guide sur Wikipédia - - Participation - - Chacun peut publier immédiatement du contenu en ligne, à condition de - respecter les règles essentielles établies par la communauté ; par - exemple, la vérifiabilité du contenu ou l’admissibilité des articles. - - De nombreuses pages d’aide sont à votre disposition, notamment pour - créer un article, modifier un article ou insérer une image. N’hésitez - pas à poser une question. - Premiers pas • L’essentiel pour contribuer - - Communauté - - 1 148 655 comptes d'utilisateur ont été créés sur Wikipédia, et, parmi - eux, 15 702 ont été actifs durant le dernier mois. Tous les rédacteurs - des articles de Wikipédia sont bénévoles ; ils constituent une - communauté collaborative, sans dirigeant, où les membres coordonnent - leurs efforts au sein des projets thématiques et de divers espaces de - discussion. - - Les débats et remarques sur les articles sont bienvenus. Les pages de - discussion servent à centraliser les réflexions et les évaluations - permettant d’améliorer les articles. - - Accueil de la communauté - - Projets frères - - - Meta-Wiki - Meta-Wiki - Coordination de tous les projets - Commons - Wikimedia Commons - Base de données multimédia - Wiktionnaire - Wiktionnaire - Dictionnaire universel - Wikinews - Wikinews - Actualités libres - Wikibooks - Wikibooks - Livres et textes didactiques - Wikiquote - Wikiquote - Recueil de citations - Wikisource - Wikisource - Bibliothèque universelle - Wikiversité - Wikiversité - Communauté pédagogique libre - Wikispecies - Wikispecies - Inventaire du vivant - - Wikipédia ne garantit pas le contenu mis en ligne. - La Wikimedia Foundation étant un hébergeur, elle ne saurait être tenue - responsable des erreurs éventuelles contenues sur ce site. - Chaque rédacteur est responsable de ses contributions. - Contact Contact - Ce document provient de « - http://fr.wikipedia.org/w/index.php?title=Wikip%C3%A9dia:Accueil_princi - pal&oldid=70578395 ». - -Outils personnels - - * Créer un compte ou se connecter - -Espaces de noms - - * Page - * Discussion - -Variantes - -Affichages - - * Lire - * Voir le texte source - * Afficher l’historique - -Actions - -Rechercher - - ____________________ (Submit) Rechercher - -Navigation - - * Accueil - * Portails thématiques - * Index alphabétique - * Article au hasard - * Contacter Wikipédia - -Contribuer - - * Premiers pas - * Aide - * Communauté - * Modifications récentes - * Faire un don - -Imprimer / exporter - - * Créer un livre - * Télécharger comme PDF - * Version imprimable - -Boîte à outils - - * Pages liées - * Suivi des pages liées - * Importer un fichier - * Pages spéciales - * Adresse de cette version - -Autres langues - - * Català - * Česky - * Deutsch - * English - * Español - * Suomi - * Magyar - * Italiano - * 日本語 - * 한국어 - * Nederlands - * Polski - * Português - * ‪Norsk (bokmål)‬ - * Română - * Русский - * Svenska - * Türkçe - * Українська - * 中文 - - * Dernière modification de cette page le 30 septembre 2011 à 12:18. - * Droit d'auteur : les textes sont disponibles sous licence Creative - Commons paternité partage à l’identique ; d’autres conditions - peuvent s’appliquer. Voyez les conditions d’utilisation pour plus - de détails, ainsi que les crédits graphiques. - Wikipedia® est une marque déposée de la Wikimedia Foundation, Inc., - organisation de bienfaisance régie par le paragraphe 501(c)(3) du - code fiscal des États-Unis. - - * Politique de confidentialité - * À propos de Wikipédia - * Avertissements - * Affichage mobile - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/fr/WINDOWS-1252/wikitop_fr_WINDOWS-1252.txt b/src/tests/testdata/fr/WINDOWS-1252/wikitop_fr_WINDOWS-1252.txt deleted file mode 100644 index 35fe742..0000000 --- a/src/tests/testdata/fr/WINDOWS-1252/wikitop_fr_WINDOWS-1252.txt +++ /dev/null @@ -1,217 +0,0 @@ - Lhistoire des Juifs en France, lune des plus anciennes communauts - juives dEurope occidentale, semble remonter au I^er sicle de lre - commune et se poursuit jusqu nos jours. - - Arrivs en Gaule peu aprs sa conqute par lEmpire romain, les Juifs - sy maintiennent sous les Mrovingiens, et connaissent une priode de - prosprit sous les Carolingiens. Au XI^e sicle, la France devient un - ple florissant de la culture juive, abritant, au nord, les communauts - ashknazes, parmi lesquelles fleurit lcole de Rachi et de ses - continuateurs, et, au sud, les Juifs de Provence et du Languedoc. La - situation se dtriore fortement aprs les Croisades, auxquelles font - suite les procs du Talmud, et les expulsions, temporaires puis - dfinitives. Un millnaire aprs leur tablissement, il ne reste plus - de Juifs que dans les tats papaux. - - Environ un sicle aprs lexpulsion des Juifs de la pninsule ibrique, - des crypto-Juifs originaires du Portugal sinstallent Bordeaux et - Bayonne. Au XVII^e sicle, les Juifs dAlsace et de Lorraine se - retrouvent eux aussi sous la juridiction de la France, suite aux - traits de Westphalie. - - Les Juifs de France sont les premiers jouir de lmancipation que la - France leur accorde au dbut de la Rvolution franaise, tant dans la - mtropole que dans les colonies. Cependant, au franco-judasme - sinscrivant dans le cadre de la lacit en France, rpond un - antismitisme la franaise , qui sexacerbe notamment lors de - laffaire Dreyfus, et sous le rgime de Vichy. Brutalement isols du - reste de la population et poursuivis avec un zle particulier par la - Milice, 75 000 Juifs meurent au cours de lOccupation, parmi lesquels - de nombreux rfugis dEurope de lEst ou dAllemagne. - - La France demeure cependant le choix naturel pour nombre de Juifs - contraints de quitter lgypte et lAfrique du Nord dans les annes - 1950 et 1960. La communaut juive de France, jusqualors - essentiellement ashknaze et assimile, devient majoritairement - sfarade et attache aux traditions. Elle est, de nos jours, la plus - importante dEurope, et comprend 483 500 personnes, qui habitent - principalement Paris, Marseille, Lyon, Nice, Toulouse et Strasbourg. - Toutes les tendances religieuses sy rencontrent, depuis les Juifs - ultra-orthodoxes jusquaux Juifs assimils. - * Lire la suite - - Contenus de qualit Bons contenus Slection Programme - - Actualits et vnements - - vnements en cours : Crise politique belge Printemps arabe Crise - de la dette dans la zone euro Crise alimentaire dans la Corne de - lAfrique Mouvement Occupy Wall Street Coupe du monde de rugby - XV Jeux panamricains - __________________________________________________________________ - - * 21 octobre : L'organisation terroriste basque ETA annonce l'arrt - dfinitif de la lutte arme, aprs plus de quarante ans d'actions - terroristes ayant caus plus de huit cents morts. - - Muammar al-Gaddafi at the AU summit.jpg - * 20 octobre : le CNT annonce la mort de Mouammar Kadhafi (photo) - lors de la prise de Syrte, dernire ville sous son contrle. - * 18 octobre : aprs cinq annes de captivit, le soldat isralien - Gilad Shalit est libr en change de la libration de - 1 027 prisonniers palestiniens. - * 13 octobre : mariage de Jigme Khesar Namgyel Wangchuck, roi du - Bhoutan, et Jetsun Pema. - * 11 octobre : lection prsidentielle au Liberia. - * 11 octobre : lex-Premier ministre et actuelle chef de lopposition - en Ukraine Ioulia Tymochenko est condamne sept ans de prison - pour abus de pouvoir. - - Octobre 2011 phmride du jour Wikinews Modifier - - Le saviez-vous ? - - vue en coupe d'un REP - * Les deux tiers des 440 racteurs nuclaires qui produisent de - l'lectricit dans le monde sont des racteurs eau pressurise - (REP). - * Le Britannique Fauja Singh dtient neuf records du monde - dathltisme pour les plus de cent ans, dont celui du marathon. - * Vakif, dans la province de Hatay, est le dernier village armnien - de Turquie. - * Depuis le 27 mars 1901, lle des Faisans, sur la Bidassoa, change - de souverainet tous les six mois entre la France et lEspagne. - * Entre 1990 et 2000, la dforestation a fait perdre la fort - amazonienne une superficie comparable celle de la France. - - Archives Modifier - - Image du jour - - - Le Quintette de Stephan, du nom de l'astronome douard Stephan, est un - groupement visuel de galaxies situ dans la constellation de Pgase. - (dfinition relle 6 064 6 760) - - Le Quintette de Stephan, du nom de l'astronome douard Stephan, est un - groupement visuel de galaxies situ dans la constellation de Pgase. - (dfinition relle 6 064 6 760 obtenir en poster) - - Archives Voir les images - Images de qualit sur Wikimdia Commons - - - Prsentation - - Wikipdia est un projet dencyclopdie collective tablie sur Internet, - universelle, multilingue et fonctionnant sur le principe du wiki. - Wikipdia a pour objectif doffrir un contenu librement rutilisable, - objectif et vrifiable, que chacun peut modifier et amliorer. - - Le cadre du projet est dfini par des principes fondateurs. Son contenu - est sous licence Creative Commons by-sa et peut tre copi et rutilis - sous la mme licence mme des fins commerciales sous rserve d'en - respecter les conditions. - - Actuellement, Wikipdia en franais compte plus de deux mille articles - distingus comme articles de qualit ou comme bons articles . - propos de Wikipdia Guide sur Wikipdia - - Participation - - Chacun peut publier immdiatement du contenu en ligne, condition de - respecter les rgles essentielles tablies par la communaut ; par - exemple, la vrifiabilit du contenu ou ladmissibilit des articles. - - De nombreuses pages daide sont votre disposition, notamment pour - crer un article, modifier un article ou insrer une image. Nhsitez - pas poser une question. - Premiers pas Lessentiel pour contribuer - - Communaut - - 1 148 655 comptes d'utilisateur ont t crs sur Wikipdia, et, parmi - eux, 15 702 ont t actifs durant le dernier mois. Tous les rdacteurs - des articles de Wikipdia sont bnvoles ; ils constituent une - communaut collaborative, sans dirigeant, o les membres coordonnent - leurs efforts au sein des projets thmatiques et de divers espaces de - discussion. - - Les dbats et remarques sur les articles sont bienvenus. Les pages de - discussion servent centraliser les rflexions et les valuations - permettant damliorer les articles. - - Accueil de la communaut - - Wikipdia ne garantit pas le contenu mis en ligne. - La Wikimedia Foundation tant un hbergeur, elle ne saurait tre tenue - responsable des erreurs ventuelles contenues sur ce site. - Chaque rdacteur est responsable de ses contributions. - Contact Contact - -Outils personnels - - * Crer un compte ou se connecter - -Espaces de noms - - * Page - * Discussion - -Variantes - -Affichages - - * Lire - * Voir le texte source - * Afficher lhistorique - -Actions - -Rechercher - - ____________________ (Submit) Rechercher - -Navigation - - * Accueil - * Portails thmatiques - * Index alphabtique - * Article au hasard - * Contacter Wikipdia - -Contribuer - - * Premiers pas - * Aide - * Communaut - * Modifications rcentes - * Faire un don - -Imprimer / exporter - - * Crer un livre - * Tlcharger comme PDF - * Version imprimable - -Bote outils - - * Pages lies - * Suivi des pages lies - * Importer un fichier - * Pages spciales - * Adresse de cette version - - * Dernire modification de cette page le 30 septembre 2011 12:18. - * Droit d'auteur : les textes sont disponibles sous licence Creative - Commons paternit partage lidentique ; dautres conditions - peuvent sappliquer. Voyez les conditions dutilisation pour plus - de dtails, ainsi que les crdits graphiques. - Wikipedia est une marque dpose de la Wikimedia Foundation, Inc., - organisation de bienfaisance rgie par le paragraphe 501(c)(3) du - code fiscal des tats-Unis. - - * Politique de confidentialit - * propos de Wikipdia - * Avertissements - * Affichage mobile diff --git a/src/tests/testdata/fr/iso-8859-1.txt b/src/tests/testdata/fr/iso-8859-1.txt new file mode 100644 index 0000000..9dfee03 --- /dev/null +++ b/src/tests/testdata/fr/iso-8859-1.txt @@ -0,0 +1,5 @@ +La norme ISO 8859-1, dont le nom complet est ISO/CEI 8859-1, et qui est souvent appele Latin-1 ou Europe occidentale, forme la premire partie de la norme internationale ISO/CEI 8859, qui est une norme de l'Organisation internationale de normalisation pour le codage des caractres en informatique. + +Elle dfinit ce qu'elle appelle l'alphabet latin numro 1, qui consiste en 191 caractres de l'alphabet latin, chacun d'entre eux tant cod par un octet (soit 8 bits). ISO 8859-1 reprend le codage des caractres imprimables d'US-ASCII. + +Dans les pays occidentaux, cette norme tait utilise par de nombreux systmes d'exploitation, dont UNIX, Windows ou AmigaOS. Elle a donn lieu quelques extensions et adaptations, dont Windows-1252 et ISO 8859-15. La distinction entre ASCII, ISO 8859-1, ISO 8859-15, Windows-1252 et MacRoman est une source de confusion parmi les dveloppeurs de programmes informatiques. Le Multinational Character Set cr par Digital Equipment Corporation pour le terminal informatique VT220 est considr comme la fois l'anctre de l'ISO 8859-1 et de l'Unicode2. Aujourd'hui, son utilisation tend dcrotre au profit de l'Unicode. diff --git a/src/tests/testdata/fr/iso-8859-15.txt b/src/tests/testdata/fr/iso-8859-15.txt new file mode 100644 index 0000000..dc76246 --- /dev/null +++ b/src/tests/testdata/fr/iso-8859-15.txt @@ -0,0 +1,16 @@ +L'uf de volaille est un produit agricole servant d'ingrdient entrant dans la +composition de nombreux plats, dans de nombreuses cultures gastronomiques du +monde. + +Le plus utilis est l'uf de poule, mais les ufs d'autres oiseaux sont aussi +consomms : caille, cane, oie, autruche, etc. Les ufs de poissons, comme le +caviar, ou de certains reptiles, comme ceux de l'iguane vert, sont galement +utiliss dans l'alimentation humaine. Cependant, leur utilisation est trs +diffrente de celle des ufs de volaille. + +Les ufs utiliss en cuisine ne sont gnralement pas fconds du fait de leur +provenance d'levages industriels o les coqs sont absents. Fconds ou non, ils +sont utiliss l'tat frais si moins de vingt-huit jours se sont couls aprs +la ponte, selon les normes administratives franaises. Dans les usages +culinaires asiatiques, les ufs sont parfois consomms couvs, comme le balut, +ou mis fermenter pendant plusieurs semaines, comme l'uf de cent ans. diff --git a/src/tests/testdata/fr/utf-16.be b/src/tests/testdata/fr/utf-16.be new file mode 100644 index 0000000..14330f0 Binary files /dev/null and b/src/tests/testdata/fr/utf-16.be differ diff --git a/src/tests/testdata/fr/utf-32.le b/src/tests/testdata/fr/utf-32.le new file mode 100644 index 0000000..13d1139 Binary files /dev/null and b/src/tests/testdata/fr/utf-32.le differ diff --git a/src/tests/testdata/fr/utf-8.txt b/src/tests/testdata/fr/utf-8.txt new file mode 100644 index 0000000..d786ca3 --- /dev/null +++ b/src/tests/testdata/fr/utf-8.txt @@ -0,0 +1,14 @@ +UTF-8 (abréviation de l’anglais Universal Character Set Transformation Format - +8 bits) est un codage de caractères informatiques conçu pour coder l’ensemble +des caractères du « répertoire universel de caractères codés », initialement +développé par l’ISO dans la norme internationale ISO/CEI 10646, aujourd’hui +totalement compatible avec le standard Unicode, en restant compatible avec la +norme ASCII limitée à l’anglais de base (et quelques autres langues beaucoup +moins fréquentes), mais très largement répandue depuis des décennies. + +L’UTF-8 est utilisé par 82,2 % des sites web en décembre 20141. De par sa +nature, UTF-8 est d’un usage de plus en plus courant sur Internet, et dans les +systèmes devant échanger de l'information. Il s’agit également du codage le plus +utilisé dans les systèmes GNU, Linux et compatibles pour gérer le plus +simplement possible des textes et leurs traductions dans tous les systèmes +d’écritures et tous les alphabets du monde. diff --git a/src/tests/testdata/fr/windows-1252.txt b/src/tests/testdata/fr/windows-1252.txt new file mode 100644 index 0000000..5f27c56 --- /dev/null +++ b/src/tests/testdata/fr/windows-1252.txt @@ -0,0 +1,3 @@ +Luf de volaille est un produit agricole servant d'ingrdient entrant dans la +composition de nombreux plats, dans de nombreuses cultures gastronomiques du +monde. diff --git a/src/tests/testdata/ga/iso-8859-1.txt b/src/tests/testdata/ga/iso-8859-1.txt new file mode 100644 index 0000000..f062a67 --- /dev/null +++ b/src/tests/testdata/ga/iso-8859-1.txt @@ -0,0 +1,6 @@ +Ag seo tarma seoltireachta a bhaineann le longa adhmaid agus le bid. + +N bhodh de cheangal idir ire agus tortha eile ach na longa, agus t ire +fin ln de lochanna agus d'aibhneacha. Fgann seo go bhfuil an teanga breac le +tarmaocht seoltireachta agus loingseoireachta agus cuid di tugtha isteach n +Lochlainnis agus n mBarla tr lonnaitheoir n iasacht. diff --git a/src/tests/testdata/ga/utf-8.txt b/src/tests/testdata/ga/utf-8.txt new file mode 100644 index 0000000..33cc012 --- /dev/null +++ b/src/tests/testdata/ga/utf-8.txt @@ -0,0 +1,6 @@ +Ag seo téarmaí seoltóireachta a bhaineann le longa adhmaid agus le báid. + +Ní bhíodh de cheangal idir Éire agus tíortha eile ach na longa, agus tá Éire +féin lán de lochanna agus d’aibhneacha. Fágann seo go bhfuil an teanga breac le +téarmaíocht seoltóireachta agus loingseoireachta agus cuid di tugtha isteach ón +Lochlainnis agus ón mBéarla trí lonnaitheoirí ón iasacht. diff --git a/src/tests/testdata/ga/windows-1252.txt b/src/tests/testdata/ga/windows-1252.txt new file mode 100644 index 0000000..1a97dae --- /dev/null +++ b/src/tests/testdata/ga/windows-1252.txt @@ -0,0 +1,6 @@ +Ag seo tarma seoltireachta a bhaineann le longa adhmaid agus le bid. + +N bhodh de cheangal idir ire agus tortha eile ach na longa, agus t ire +fin ln de lochanna agus daibhneacha. Fgann seo go bhfuil an teanga breac le +tarmaocht seoltireachta agus loingseoireachta agus cuid di tugtha isteach n +Lochlainnis agus n mBarla tr lonnaitheoir n iasacht. diff --git a/src/tests/testdata/he/UTF-8/wikitop_he_UTF-8.txt b/src/tests/testdata/he/UTF-8/wikitop_he_UTF-8.txt deleted file mode 100644 index 1331912..0000000 --- a/src/tests/testdata/he/UTF-8/wikitop_he_UTF-8.txt +++ /dev/null @@ -1,185 +0,0 @@ - #ויקיפדיה copyright Atom של ויקיפדיה - -עמוד ראשי - - מתוך ויקיפדיה, האנציקלופדיה החופשית - קפיצה אל: ניווט, חיפוש - - ברוכים הבאים לוויקיפדיה! - - ויקיפדיה היא מיזם רב לשוני לחיבור אנציקלופדיה שיתופית, חופשית ומהימנה, - שכולם יכולים לערוך. כעת יש בוויקיפדיה העברית 125,392 ערכים. - - P mathematics.svg מתמטיקה P physics.svg פיזיקה P chemistry.svg כימיה - Q space.svg מדעי החלל P technology.png טכנולוגיה P biology.svg ביולוגיה - P medicine.svg רפואה P geography1.png גאוגרפיה P Society.png מדעי החברה - P Economy.png כלכלה P parthenon.svg מדע המדינה P La Liberte.png - היסטוריה - P religion world.svg דת P judaism.svg יהדות P Israel Flag2.png ישראל - P treble clef.svg מוזיקה P literature.svg ספרות P vip.svg אישים - P SPORT.png ספורט P art.png אמנות P countries.png מדינות העולם - פורטל היום - P Football.png - - פורטל הכדורגל הוא שער לכל הנושאים בוויקיפדיה הקשורים בכדורגל. בפורטל - נמצאים קישורים לערכים ולתמונות נבחרות בנושא, לאישים בולטים בתחום, - למושגים חשובים ולנושאי משנה. - פורטלים אקראיים - (כל הפורטלים | טוען פורטלים...) P art.png אמנות P physics.svg פיזיקה - P geography1.png גאוגרפיה P chemistry.svg כימיה - תמונת היום - תמונת היום - [זיקית המתאימה את צבע עורה לסביבתה, למטרת הסוואה. - - זיקית המתאימה את צבע עורה לסביבתה, למטרת הסוואה, יכולת שבה ידועה הזיקית - במיוחד. הזיקיות נפוצות בעיקר באפריקה, ובאי מדגסקר מצוי המגוון הרחב - ביותר של מיני זיקיות, וכן היא נפוצה באזורים טרופיים נוספים בעולם. - הזיקיות בדרך כלל שוכנות עצים או שיחים, אם כי ישנם מינים של זיקיות קטנות - השוכנות על הקרקע תחת מחסה עלי השלכת. - תמונות מומלצות נוספות - תמונה מומלצת אקראית - ערך מומלץ - ערך מומלץ - - אחריות נכסים או שעבוד נכסים, הוא מונח הלכתי-משפטי, המתאר שעבוד של נכסים - או אחריות של אדם, היוצרים ערבות משפטית בתחומים שונים. דוגמה קלאסית - לאחריות נכסים היא בלוויית כסף, בה לעתים ניתן לגבות את החוב מהנכסים שהיו - שייכים ללווה בזמן שבוצעה ההלוואה אף אם הם כבר נמכרו לאחר, מכיוון שעוד - משעת ההלוואה חל על נכסיו שעבוד כלפי החוב. המונח המקביל לשעבוד במשפט - הכללי הוא "שעבוד צף", ששונה באופן מהותי מהמונח ההלכתי בכך שהוא חל רק על - נכסי חברות ולא על נכסי יחידים. - - סוג מסוים של אחריות נכסים הוא ה"אַפּוֹתֵּיקֵי", במסגרתו השעבוד שחל על - נכסי החייב מתייחס לנכס מסוים אחד, שממנו נגבה החוב. המונח המקביל לשעבוד - מסוג זה במשפט הכללי הוא שעבוד מסוג "בטוחה". - לערך המלא - מומלצים נוספים - הידעת? - הידעת? - אליעזר קפלן, שר האוצר הראשון של ישראל - - בחלקת גדולי האומה שבהר הרצל שבירושלים קבורים גדולי המנהיגים של מדינת - ישראל. בין השאר זכאים להיקבר בבית העלמין נשיאי מדינת ישראל, ראשי - הממשלה, יושבי ראש הכנסת ובני זוגם וכן מעוטרי עיטור הגבורה. עם זאת, בשני - מקרים חרגו מכלל זה: שר האוצר הראשון אליעזר קפלן היה לראשון להיקבר - בחלקה, לאחר שנפטר בעת כהונתו ובטרם נקבעו אמות המידה לזכאות. ב-2007 - הוחלט לקבור במקום גם את ראש עיריית ירושלים טדי קולק. מנגד, יש זכאים - שבחרו להיקבר במקום אחר, כגון נשיאה הרביעי של ישראל אפרים קציר, שביקש - להיקבר בבית הקברות שברחובות לצד רעייתו, וראש ממשלתה השישי מנחם בגין, - שביקש להיקבר בחלקת עולי הגרדום שבהר הזיתים. - לקטעי "הידעת?" נוספים - קטע "הידעת?" אקראי - היום בהיסטוריה - היום בהיסטוריה - שעון מתקופת לוח השנה המהפכני בצרפת, שבו חולק היום בחלוקה עשרונית - * 1648 - נחתמים הסכמי וסטפליה, המביאים לסיומה של מלחמת שלושים השנים - * 1793 - האספה הלאומית של צרפת מקבלת את לוח השנה המהפכני בצרפת - * 1929 - יום חמישי השחור - התמוטטות הבורסה של ניו יורק, יחד עם - התמוטטות נוספת ביום שלישי השחור, ה-29 באוקטובר, מסמנת את תחילת השפל - הגדול - * 1945 - וידקון קוויזלינג מוצא להורג בנורבגיה בעוון בגידה - * 1964 - זמביה זוכה לעצמאותה מהאימפריה הבריטית - * 1973 - הפסקת האש בין ישראל, מצרים וסוריה נכנסת לתוקפה ומביאה לסיום - מלחמת יום הכיפורים - - 24 באוקטובר - חודש אוקטובר - היום בהיסטוריה - אירועים בלוח העברי - אירועים בלוח העברי - בניין האו"ם בניו יורק - * ה'תרנ"ו - נולדה בבה אידלסון, חברת כנסת, ממנהיגות תנועת העבודה - וממייסדות מועצת הפועלות. - * ה'תש"ט - תחילת מבצע חירם, לשחרור הגליל העליון, במלחמת העצמאות. - * ה'תשל"ד - מועצת הביטחון של האו"ם מאשרת את החלטה מס' 338. - - פרשת השבוע: נח, שבת ראש חודש - כ"ו בתשרי ה'תשע"ב - אירועים בלוח העברי - חדשות ואקטואליה - חדשות ואקטואליה - כריסטינה פרננדס דה קירשנר - * מאות הרוגים ברעידת אדמה בעוצמה 7.2 בסולם ריכטר שפקדה את מזרח טורקיה - (‏ynet) - * בבחירות לנשיאות ארגנטינה זכתה הנשיאה המכהנת כריסטינה פרננדס דה - קירשנר; החוק בארגנטינה לא מאפשר יותר משתי כהונות רצופות, לכן לא - תוכל פרננדס דה קירשנר לשוב ולהתמודד בבחירות הבאות (נענע10) - * בתפילת שחרית של שבת בשטיבלך מאה שערים פרצה קטטה אלימה בין חסידי גור - לבין סיקריקים אשר במהלכה נפגע אחד המתפללים, דבר שגרם להתערבות - המשטרה (כיכר השבת) - - עריכה - מיזמי ויקימדיה אחרים - מיזמי ויקימדיה אחרים - - - ללא הודעת הגנה אוטומטית - מקור: - http://he.wikipedia.org/w/index.php?title=%D7%A2%D7%9E%D7%95%D7%93_%D7% - A8%D7%90%D7%A9%D7%99&oldid=11356476 - -כלים אישיים - - * כניסה לחשבון / הרשמה - -גרסאות שפה - -מרחבי שם - - * שיחה - * ערך - -חיפוש - - (Submit) חיפוש ____________________ - -פעולות - -צפיות - - * גרסאות קודמות - * הצגת מקור - * קריאה - -ניווט - - * עמוד ראשי - * ברוכים הבאים - * שינויים אחרונים - * ערכים מומלצים - * פורטלים - * ערך אקראי - * תרומה לוויקיפדיה - -קהילה - - * שער הקהילה - * עזרה - * ייעוץ - * מזנון - * לוח מודעות - * יצירת קשר - * ספר אורחים - -תיבת כלים - - * דפים המקושרים לכאן - * שינויים בדפים המקושרים - * דפים מיוחדים - * קישור קבוע - * ציטוט דף זה - -דף זה בשפות אחרות - - -הדפסה/יצוא - - * יצירת ספר - * הורדה כ־PDF - * גרסת הדפסה - - * שונה לאחרונה ב־14:43, 14 באוקטובר 2011. (ציטוט) - * הטקסט מוגש בכפוף לרישיון CC ייחוס-שיתוף זהה 3.0; פרטי הרישוי של - התמונות מופיעים בעמודי התמונות. ראו תנאי שימוש לפרטים נוספים. - - * מדיניות הפרטיות - * אודות ויקיפדיה - * הבהרה משפטית - * תצוגה למכשירים ניידים - diff --git a/src/tests/testdata/he/WINDOWS-1255/wikitop_he_WINDOWS-1255.txt b/src/tests/testdata/he/WINDOWS-1255/wikitop_he_WINDOWS-1255.txt deleted file mode 100644 index 174805d..0000000 --- a/src/tests/testdata/he/WINDOWS-1255/wikitop_he_WINDOWS-1255.txt +++ /dev/null @@ -1,185 +0,0 @@ - # copyright Atom - - - - , - : , - - ! - - , , - . 125,392 . - - P mathematics.svg P physics.svg P chemistry.svg - Q space.svg P technology.png P biology.svg - P medicine.svg P geography1.png P Society.png - P Economy.png P parthenon.svg P La Liberte.png - - P religion world.svg P judaism.svg P Israel Flag2.png - P treble clef.svg P literature.svg P vip.svg - P SPORT.png P art.png P countries.png - - P Football.png - - . - , , - . - - ( | ...) P art.png P physics.svg - P geography1.png P chemistry.svg - - - [ , . - - , , - . , - , . - , - . - - - - - - , -, - , . - , - , - . - " ", - . - - "", - , . - "". - - - ? - ? - , - - - . , - , . , - : - , . -2007 - . , - , , - , , - . - "?" - "?" - - - , - * 1648 - , - * 1793 - - * 1929 - - , - , -29 , - - * 1945 - - * 1964 - - * 1973 - , - - - 24 - - - - - " - * '" - , , - . - * '" - , , . - * '" - " ' 338. - - : , - " '" - - - - - * 7.2 - (ynet) - * - ; , - (10) - * - , - ( ) - - - - - - - - : - http://he.wikipedia.org/w/index.php?title=%D7%A2%D7%9E%D7%95%D7%93_%D7% - A8%D7%90%D7%A9%D7%99&oldid=11356476 - - - - * / - - - - - - * - * - - - - (Submit) ____________________ - - - - - - * - * - * - - - - * - * - * - * - * - * - * - - - - * - * - * - * - * - * - * - - - - * - * - * - * - * - - - - -/ - - * - * PDF - * - - * 14:43, 14 2011. () - * CC - 3.0; - . . - - * - * - * - * - diff --git a/src/tests/testdata/he/iso-8859-8.txt b/src/tests/testdata/he/iso-8859-8.txt new file mode 100644 index 0000000..bc78eee --- /dev/null +++ b/src/tests/testdata/he/iso-8859-8.txt @@ -0,0 +1,2 @@ + ISO 8859 , , : + 0xA0 0xBF , , . diff --git a/src/tests/testdata/he/utf-8.txt b/src/tests/testdata/he/utf-8.txt new file mode 100644 index 0000000..1fbe418 --- /dev/null +++ b/src/tests/testdata/he/utf-8.txt @@ -0,0 +1,3 @@ +(ראשי תיבות של 8‎-bit Unicode Transformation Format או 8‎-bit UCS Transformation Format) הוא קידוד תווים באורך משתנה ליוניקוד, שנוצר על ידי רוב פייק וקן תומפסון. ניתן לקודד בו כל תו המצוי בתקן יוניקוד על ידי שימוש באחד עד ארבעה בתים, תלוי בתו. הקידוד ב-UTF-8 מעניק את כל יתרונות השימוש בקידוד ליוניקוד ומוסיף עליהם, בין היתר, גם חיסכון בזיכרון, עמידות בפני איבוד או השחתת בתים ותאימות לאחור ל-ASCII. ה-IETF מעדיף בבירור את UTF-8 ומחייב כל פרוטוקול אינטרנט לתמוך בו, וכן קונסורציום הדואר האלקטרוני, ה-IMC, ממליץ שכל תוכנת דואר אלקטרוני תוכל להציג וליצור דואר באמצעות UTF-8. + + diff --git a/src/tests/testdata/he/windows-1255.txt b/src/tests/testdata/he/windows-1255.txt new file mode 100644 index 0000000..d915bd4 --- /dev/null +++ b/src/tests/testdata/he/windows-1255.txt @@ -0,0 +1 @@ + , -, , . diff --git a/src/tests/testdata/hr/ibm852.txt b/src/tests/testdata/hr/ibm852.txt new file mode 100644 index 0000000..98f5138 --- /dev/null +++ b/src/tests/testdata/hr/ibm852.txt @@ -0,0 +1,4 @@ +Brekinja (lat. Sorbus torminalis) je bjelogorina vrsta drvea iz porodice +Rosaceae. +Prirodno je rasprostranjena u zapadnoj, srednjoj i junoj Europi, sjevernoj +Africi, Krimu, Maloj Aziji, Kavkazu i Transkavkaziji. diff --git a/src/tests/testdata/hr/iso-8859-13.txt b/src/tests/testdata/hr/iso-8859-13.txt new file mode 100644 index 0000000..124b1c0 --- /dev/null +++ b/src/tests/testdata/hr/iso-8859-13.txt @@ -0,0 +1,4 @@ +Brekinja (lat. Sorbus torminalis) je bjelogorina vrsta drvea iz porodice +Rosaceae. +Prirodno je rasprostranjena u zapadnoj, srednjoj i junoj Europi, sjevernoj +Africi, Krimu, Maloj Aziji, Kavkazu i Transkavkaziji. diff --git a/src/tests/testdata/hr/iso-8859-16.txt b/src/tests/testdata/hr/iso-8859-16.txt new file mode 100644 index 0000000..a10a108 --- /dev/null +++ b/src/tests/testdata/hr/iso-8859-16.txt @@ -0,0 +1,4 @@ +Brekinja (lat. Sorbus torminalis) je bjelogorina vrsta drvea iz porodice +Rosaceae. +Prirodno je rasprostranjena u zapadnoj, srednjoj i junoj Europi, sjevernoj +Africi, Krimu, Maloj Aziji, Kavkazu i Transkavkaziji. diff --git a/src/tests/testdata/hr/iso-8859-2.txt b/src/tests/testdata/hr/iso-8859-2.txt new file mode 100644 index 0000000..7c33320 --- /dev/null +++ b/src/tests/testdata/hr/iso-8859-2.txt @@ -0,0 +1,4 @@ +Brekinja (lat. Sorbus torminalis) je bjelogorina vrsta drvea iz porodice +Rosaceae. +Prirodno je rasprostranjena u zapadnoj, srednjoj i junoj Europi, sjevernoj +Africi, Krimu, Maloj Aziji, Kavkazu i Transkavkaziji. diff --git a/src/tests/testdata/hr/maccentraleurope.txt b/src/tests/testdata/hr/maccentraleurope.txt new file mode 100644 index 0000000..27c41e0 --- /dev/null +++ b/src/tests/testdata/hr/maccentraleurope.txt @@ -0,0 +1,4 @@ +Brekinja (lat. Sorbus torminalis) je bjelogorina vrsta drvea iz porodice +Rosaceae. +Prirodno je rasprostranjena u zapadnoj, srednjoj i junoj Europi, sjevernoj +Africi, Krimu, Maloj Aziji, Kavkazu i Transkavkaziji. diff --git a/src/tests/testdata/hr/utf-8.txt b/src/tests/testdata/hr/utf-8.txt new file mode 100644 index 0000000..92cc1dd --- /dev/null +++ b/src/tests/testdata/hr/utf-8.txt @@ -0,0 +1,4 @@ +Brekinja (lat. Sorbus torminalis) je bjelogorična vrsta drveća iz porodice +Rosaceae. +Prirodno je rasprostranjena u zapadnoj, srednjoj i južnoj Europi, sjevernoj +Africi, Krimu, Maloj Aziji, Kavkazu i Transkavkaziji. diff --git a/src/tests/testdata/hr/windows-1250.txt b/src/tests/testdata/hr/windows-1250.txt new file mode 100644 index 0000000..60d8c98 --- /dev/null +++ b/src/tests/testdata/hr/windows-1250.txt @@ -0,0 +1,4 @@ +Brekinja (lat. Sorbus torminalis) je bjelogorina vrsta drvea iz porodice +Rosaceae. +Prirodno je rasprostranjena u zapadnoj, srednjoj i junoj Europi, sjevernoj +Africi, Krimu, Maloj Aziji, Kavkazu i Transkavkaziji. diff --git a/src/tests/testdata/hu/.WINDOWS-1250/wikitop_hu_WINDOWS-1250.txt b/src/tests/testdata/hu/.WINDOWS-1250/wikitop_hu_WINDOWS-1250.txt deleted file mode 100644 index 9473dea..0000000 --- a/src/tests/testdata/hu/.WINDOWS-1250/wikitop_hu_WINDOWS-1250.txt +++ /dev/null @@ -1,388 +0,0 @@ -Kezdlap - - A Wikipdibl, a szabad enciklopdibl - Ez a lap egy ellenrztt vltozata rszletek megjelentse/elrejtse - Ez a kzztett vltozat, ellenrizve: 2011. mrcius 31. - - Pontossg ellenrztt - - Ugrs: navigci, keress - - dvzlnk a Wikipdiban! - Ezt az enciklopdit az olvasi szerkesztik - A magyar vltozatnak 202 963 szcikke van, - ebbl 623 kiemelt. - - * Trsadalom - * Trtnelem - * Fldrajz - - * Kultra - * Mvszet - * Sport - - * Termszet - * Fizika - * Biolgia - - * Matematika - * Technika - * sszes portl - - Mobilos vltozatKapcsolatfelvtel / Contact - - Kiemelt cikk - - Liszt-kaulbach.jpg - - Liszt Ferenc (nmetl Franz Liszt) (Doborjn, 1811. oktber 22. - Bayreuth, 1886. jlius 31.) a 19. szzad egyik legjelentsebb - romantikus zeneszerzje, minden idk egyik legnagyobb zongoramvsze. - - Apja, Liszt dm, Esterhzy herceg uradalmi hivatalnoka, korn - felismerte finak kivteles zenei tehetsgt, s minden lehetsget - megragadott kibontakoztatsra. Liszt kilencves korban mr - nyilvnossg eltt zongorzott Sopronban s Pozsonyban, majd mprtol - furak tmogatsval hamarosan Bcsben folytathatta tanulmnyait Czerny - s Salieri tantvnyaknt. 1822. december 1-jn mutatkozott be az - osztrk fvrosban. Els, nagy feltnst kelt bcsi koncertjn az - akkor 11 ves mvszre mg Beethoven is felfigyelt. Itt jelent meg - nyomtatsban els mve, varicii ugyanarra a Diabelli-tmra - (keringre), amelyre Beethoven hres zongoramvt rta. Az ifj Liszt - bcsi szereplse, majd ngy nagy siker pesti, illetve budai koncertje - utn 1823 szn atyjval az eurpai mvszeti let kzpontjba, - Prizsba utazott. - - Felvtelt a prizsi Conservatoire-ba Cherubini, az intzmny vezetje - gtolta meg, ezrt Bcsben maradt, ahol zeneelmletet s ellenpontot - tanult magnton, kzben hangversenykrtra indult. Londoni - bemutatkozst szmos jabb meghvs kvette Franciaorszgba, Svjcba, - majd ismt Angliba. Msorn tbbnyire a npszer operkra rt - brndok, parafrzisok szerepeltek. 1827-ben egy idre a francia - fvrosban telepedett le, ahol tehetsgnek ksznheten a trsasgi - let nnepelt hse lett. Itt ismerkedett meg Chopinnal, akitl - eltanulta a zongorajtk rnyalatait, valamint Berliozzal, akitl a - zenekari nyelvet s a programzene j vvmnyait sajttotta el. Az - 1830-as vek vgn jabb hangversenykrutakra indult, bejrta egsz - Eurpt Portuglitl Oroszorszgig. 1835-ben a genfi konzervatrium - tanra lett. Az 1838-as pesti rvzkatasztrfa krosultjait Bcsben - rendezett hangversenyeinek jvedelmvel tmogatta. Ugyanilyen bkezen - jrult hozz nhny vvel ksbb a bonni Beethoven-emlkm - ltrehozshoz. Az oroszorszgi hangversenykrutakon ismerte meg az - orosz komponistk mvszett, valamint szerelmt, Carolyne zu - Sayn-Wittgenstein hercegnt, aki a ksbb irodalmi munkssgban is - kzremkdtt. A negyvenes vek elejn Weimarban az udvari sznhz - karmestere lett: npszerstette s veznyelte kortrsai mveit, - tantotta a kontinens minden sarkbl rkez plyakezd muzsikusokat s - komponistkat. Liszt lete utols vtizedeit Weimar, Rma s Budapest - kztt osztotta meg. A budapesti Zeneakadmia els veiben kszsggel - segtette a magyar zenekultra jonnan kialakul kzpontjt, s az v - nhny hnapjban tantott is itt. - - Tovbb a szcikkhez - __________________________________________________________________ - - A kezdlapon legutbb megjelent szcikkek: I. Kroly magyar kirly - Aerosmith Giuseppe Verdi Harisnya Kznsges cserepestekns - Nrnbergi per Bartk Bla - - Tovbbi kiemelt lapok Kiemelt szcikk RSS - - - A nap kpe - - Kznsges csimpnz (Pan troglodytes) - - Kznsges csimpnz (Pan troglodytes) - - Teljes mret Archvum Kiemelt kp RSS - -Tudtad? - - Tudtad-e, hogy - * a msodik vilghbor utni Budapesten egy villamoskocsiban - knyvtrat mkdtettek? - * a knai csszrokat, majd Mao Ce-tungot is a - vanszuj-kszntssel dvzltk, aminek jelentse tzezer v, - s amibl a japn banzj sz is ered? - * a dzsihd t jelentse kzl csak az egyik a szent hbor s nem - is a legfontosabb, az elsdleges a szv/llek dzsihdja (dzsihd - bin nafsz/kalb): bels kzdelem a gonoszsg ellen? - * Kanada neve a Szent Lrinc-folynl beszlt irokz kanata szbl - ered, melynek jelentse falu, telepls? - - - Aktulis - - Az Emlkm - Az Emlkm - Transtrmer - Transtrmer - Dan Shechtman - Dan Shechtman - Steve Jobs - Steve Jobs - * Oktber 21-n Szojuz raktval plyra lltottk az eurpai - Galileo navigcis rendszer els kt mholdjt. - * Oktber 16-n felavattk Martin Luther King Nobel-bkedjas - polgrjogi harcos 9 mteres szobrt is magba foglal emlmvt - Washington szvben a National Mall nemzeti Emlkparkban George - Washington s Abraham Lincoln emlkmve mellett. - * Oktber 15-n felavattk Boldog II. Jnos Pl els szobrt - Oroszorszgban, melyet a moszkvai nemzetkzi irodalmi knyvtr - udvarn lltottak fel. - * Oktber 11-n elhunyt Mszly Dezs Kossuth-djas magyar r, - klt, mfordt, dramaturg, Villon s Shakespeare mveinek avatott - tolmcsolja. - * Oktber 10-n, a Nobel-djas ht utols napjn a 2011-es - kzgazdasgi Nobel-emlkdjat Thomas J. Sargent s Christopher A. - Sims rszre tltk oda a gazdasgi folyamatokban az okok s - hatsok klnvlasztsrl szl elmleteikrt. - * Oktber 7-n, a Nobel-djas ht tdik napjn a 2011-es - Nobel-bkedjat Ellen Johnson-Sirleaf libriai elnk, Leymah Gbowee - libriai bkeaktivista s Tavakkul Karmn jemeni jogvd rszre - tltk oda megosztva a nk jogainak vdelmrt jelentettk be az - osli Norvg Nobel Intzetben. - * Oktber 6-n, a Nobel-djas ht negyedik napjn az idei irodalmi - Nobel-djat Tomas Transtrmer svd r, klt, mfordt rszre - tltk oda pontos, letisztult klti kpeirt, amelyek rvn j - fnyben lthatjuk a valsgot. - * Oktber 5-n, a Nobel-djas ht harmadik napjn az idei kmiai - Nobel-djat Daniel Shechtman kutat rszre tltk oda a - kvzikristlyok felfedezsrt. - * Oktber 5-n elhunyt Steve Jobs, az Apple Inc. elnke, korbbi - vezrigazgatja, az informatikai s a szrakoztat ipar kiemelked - alakja. - * Oktber 4-n, a Nobel-djas ht msodik napjn az idei fizikai - Nobel-djat Saul Perlmutter, Brian P. Schmidt s Adam G. Riess - csillagszoknak megosztva tltk oda a vilgegyetem gyorsul - temben val tgulsnak tvoli szupernvk megfigyelsvel trtnt - felfedezsrt. - * Oktber 3-n megkezddtt a 2011. vben Nobel-djjal kitntettek - nvsornak kihirdetse; az lettani-orvosi Nobel-dj felt Ralph - Steinman rszre tltk oda a dendritikus sejtekkel kapcsolatos - munkssgrt (a djazott a kihirdets eltt nhny nappal hunyt - el), msik felt megosztva veheti t december 10-n Bruce A. - Beutler s Jules Hoffmann az gynevezett termszetes vagy - veleszletett immunrendszer kutatsrt. - __________________________________________________________________ - - Hallesetek a kzelmltban - - - Ezen a napon - - Ma 2011. oktber 21. van, - * Orsolya nvnapja - - Nobel Alfrd - Nobel Alfrd - Tra-tekercs - Tra-tekercs - * Alfred Nobel nnepnapja - * Fldnkrt vilgnap (Magyarorszgon 1992 ta) - * Szent Orsolya s vrtantrsainak emlknapja - * Szimchat Tr (a Tra rmnnepe, napnyugtig) - * Boldog IV. Kroly kirly emlknapja (2004 ta) - * almanap, az almafk s a gymlcssk napja az Egyeslt - Kirlysgban - * a nachos nemzetkzi napja Mexikban s az Amerikai Egyeslt - llamokban - * polk napja Thaifldn - * tengerentli knaiak napja Tajvanon - * Trafalgar-nap az Egyeslt Kirlysgban - * Dizzy Gillespie szletsnapja - - vfordulk - * 247 ve, 1764-ben szletett Bihari Jnos cigny szrmazs magyar - zeneszerz s hegedmvsz, a Rkczi-indul vlelmezett szerzje, - akinek virtuz jtkt Liszt Ferenc is hallgatta s nagy - elismerssel emlkezett meg rla ( 1827). - * 150 ve, 1861-ben szletett Klupathy Jen fizikus, az MTA tagja, a - gyakorlati fizika kiemelked alakja( 1931). - * 140 ve, 1871-ben szletett Plfy Mric geolgus, hidrogeolgus, - egyetemi magntanr; a budapesti Fldtani Intzet fgeolgusa, a - Magyarhoni Fldtani Trsulat titkra, majd elnke, az MTA tagja - ( 1930). - * 100 ve, 1911-ben kttt hzassgot HabsburgLotaringiai Kroly - fherceg s Zita Bourbonprmai hercegn (1916-tl IV. Kroly - magyar kirly s Zita magyar kirlyn, illetve I. Kroly osztrk - csszr s Zita osztrk csszrn). - * 80 ve, 1931-ben hunyt el Arthur Schnitzler osztrk r, drmar - (* 1862) . - * 70 ve, 1941-ben szletett Jankovics Marcell Kossuth-djas - rajzfilmrendez, knyvillusztrtor, kultrtrtnsz, - kultrpolitikus. - - Jankovics Marcell - Jankovics Marcell - Arthur Schnitzler - Arthur Schnitzler - Hzassgkts - Hzassgkts - Bihari Jnos - Bihari Jnos - - Tovbbi nnepek nnepek RSS Tovbbi vfordulk vfordulk RSS - - - Ismerkeds a Wikipdival - - * dvzllap Bemutatkozik a Wikipdia. - * Els lpsek Didaktikusan felptett bevezet a Wikipdia - szerkesztsbe. - * Segtsg Egyszer, kzepes s halad szerkesztsi tippek, - tmutatk. - * Wikifogalmak Minden, ami elsre knaiul hangzik a Wikipdin, itt - magyarzatra lel. - * Kocsmafal kezdknek Az j szerkeszt krdez, aki tud, vlaszol. - * Homokoz jdonslt szerkesztink szabadon garzdlkodhatnak - benne. - * Mentorls Kezdeti lpseidhez tmogatst kaphatsz egy - tapasztaltabb szerkeszttl. - - Szerkeszti kzssg - - * Kzssgi portl A magyar Wikipdia szerkeszti kzssgnek - virtulis dolgozszobja. - * Kapcsolatfelvtel / Contact Kapcsolatfelvteli lehetsgek - ttekintse. - * Nagykvetsg / Embassy Kapcsolat ms nyelv Wikipdikkal. - * zenfal A Wikipdia klnbz tisztsgviselihez intzhet - krsek s krdsek platformja. - * Adminisztrtorok zenfala Az adminisztrtori beavatkozst - ignyl rendkvli esetekre. - * Wikipdia-IRC Instant csevej, rtekezlet lben a tbbi - szerkesztvel (belps). - * Esemnyek - A Wikipdia letben soron kvetkez esemnyek. - - - Wikipdia ms nyelveken - - Ez a magyar nyelv Wikipdia. 2003-ban indult, jelenleg 202 963 - szcikket tartalmaz. A Wikipdia sok ms nyelven is elrhet. - * A legnagyobb Wikipdik (tbb mint 500 000 szcikkel): angol, - francia, holland, japn, lengyel, nmet, olasz, orosz, portugl, - spanyol. - - * Wikipdik a Magyarorszgon hivatalosan elismert nemzetisgek - nyelvein: bolgr, cigny, grg, horvt, lengyel, nmet, rmny, - romn, ruszin, szerb, szerbhorvt, szlovk, szlovn, ukrn. - - Tovbbi nyelvek (magyarul) Tovbbi nyelvek (angolul, legfrissebb) - -Wikimdia-trslapok - - A Wikipdit a nonprofit Wikimdia Alaptvny zemelteti. A Wikimdia - szmos tbbnyelv s nylt tartalm trslapot zemeltet: - - Wikisztr Wikisztr - Tbbnyelv sztr s szinonimasztr Wikidzet Wikidzet - Tbbnyelv idzet- s szlsgyjtemny Wikifajok Wikifajok - Rendszertani adatbzis Wikiknyvek Wikiknyvek - Szabad kziknyvek s tmutatk - Wikimedia Commons Wikimedia Commons - Szabad mdiallomnyok gyjtemnye Wikiegyetem Wikiegyetem - Jegyzetek s tanulsi segdletek Wikiforrs Wikiforrs - Szabad forrsmunkk Meta-Wiki Meta-Wiki - A Wikimedia projektek koordincija - - Felhasznlsi felttelek - - A Wikipdiban tallhat szvegekre s egyes kpekre a Creative Commons - Nevezd meg! gy add tovbb! 3.0 (CC-BY-SA-3.0) licenc vonatkozik. - Cc.logo.circle.svg - Cc-by new.svg - Cc-sa.svg - * Minden szerkesztsed ezen licenc elfogadst s alkalmazst - jelenti, mellyel hozzjrulsz, hogy a mvet brki mdosthatja, azt - brmilyen clra felhasznlhatja. - * A Wikipdia tartalmnak jrafelhasznlsakor (tdolgozs esetn is) - meg kell adnod, hogy a mre a fenti licenc vonatkozik, tovbb meg - kell nevezned a forrsul szolgl szcikket; internetes kzzttel - esetn a forrsmre mutat linket kell elhelyezned honlapodon. - * A kpek felhasznlsi feltteleit kln kell ellenrizned. - - Felhasznlsi felttelek Jogi nyilatkozat jrafelhasznlsi - tmutat - A lap eredeti cme: - http://hu.wikipedia.org/w/index.php?title=Kezd%C5%91lap&oldid=9469813 - Kategria: - * Hibs fjlhivatkozsokat tartalmaz lapok - - Rejtett kategria: - * Kezdlap - - Mit gondolsz errl az oldalrl? - - Arra krnk, sznj egy percet a cikk rtkelsre! A visszajelzsek - segtenek az oldal fejlesztsben. - Megbzhatsg: [(bizonytalan)] Teljessg: [(bizonytalan)] - Trgyilagossg: [(bizonytalan)] Stlus: [(bizonytalan)] Klds - ____________ - -Szemlyes eszkzk - - * Bejelentkezs / fik ltrehozsa - -Nvterek - - * Kezdlap - * Vitalap - -Vltozk - -Nzetek - - * Olvass - * Lapforrs - * Laptrtnet - -Mveletek - -Keress - - ____________________ (Submit) Keress - -Navigci - - * Kezdlap - * Tartalom - * Kiemeltek - * Friss vltoztatsok - * Lap tallomra - * Tudakoz - -Rszvtel - - * Kezdknek - * Segtsg - * Kzssgi portl - * Kapcsolatfelvtel - * Adomnyok - -Nyomtats/exportls - - * Knyv ksztse - * Letlts mint PDF - * Nyomtathat vltozat - -Eszkzk - - * Mi hivatkozik erre - * Kapcsold vltoztatsok - * Specilis lapok - * Link erre a vltozatra - * Hogyan hivatkozz erre a lapra - * Lap rtkelse - -Ms nyelveken - - * A lap utols mdostsa: 2011. mrcius 31., 21:44 - * A lap szvege Creative Commons Nevezd meg! gy add tovbb! 3.0 - licenc alatt van; egyes esetekben ms mdon is felhasznlhat. - Rszletekrt lsd a felhasznlsi feltteleket. - - * Adatvdelmi irnyelvek - * A Wikipdirl - * Jogi nyilatkozat diff --git a/src/tests/testdata/hu/ISO-8859-2/wikitop_hu_ISO-8859-2.txt b/src/tests/testdata/hu/ISO-8859-2/wikitop_hu_ISO-8859-2.txt deleted file mode 100644 index 7f326df..0000000 --- a/src/tests/testdata/hu/ISO-8859-2/wikitop_hu_ISO-8859-2.txt +++ /dev/null @@ -1,444 +0,0 @@ -Kezdlap - - A Wikipdibl, a szabad enciklopdibl - Ez a lap egy ellenrztt vltozata rszletek megjelentse/elrejtse - Ez a kzztett vltozat, ellenrizve: 2011. mrcius 31. - - Pontossg ellenrztt - - Ugrs: navigci, keress - - dvzlnk a Wikipdiban! - Ezt az enciklopdit az olvasi szerkesztik - A magyar vltozatnak 202 963 szcikke van, - ebbl 623 kiemelt. - - * Trsadalom - * Trtnelem - * Fldrajz - - * Kultra - * Mvszet - * Sport - - * Termszet - * Fizika - * Biolgia - - * Matematika - * Technika - * sszes portl - - Mobilos vltozatKapcsolatfelvtel / Contact - - Kiemelt cikk - - Liszt-kaulbach.jpg - - Liszt Ferenc (nmetl Franz Liszt) (Doborjn, 1811. oktber 22. - Bayreuth, 1886. jlius 31.) a 19. szzad egyik legjelentsebb - romantikus zeneszerzje, minden idk egyik legnagyobb zongoramvsze. - - Apja, Liszt dm, Esterhzy herceg uradalmi hivatalnoka, korn - felismerte finak kivteles zenei tehetsgt, s minden lehetsget - megragadott kibontakoztatsra. Liszt kilencves korban mr - nyilvnossg eltt zongorzott Sopronban s Pozsonyban, majd mprtol - furak tmogatsval hamarosan Bcsben folytathatta tanulmnyait Czerny - s Salieri tantvnyaknt. 1822. december 1-jn mutatkozott be az - osztrk fvrosban. Els, nagy feltnst kelt bcsi koncertjn az - akkor 11 ves mvszre mg Beethoven is felfigyelt. Itt jelent meg - nyomtatsban els mve, varicii ugyanarra a Diabelli-tmra - (keringre), amelyre Beethoven hres zongoramvt rta. Az ifj Liszt - bcsi szereplse, majd ngy nagy siker pesti, illetve budai koncertje - utn 1823 szn atyjval az eurpai mvszeti let kzpontjba, - Prizsba utazott. - - Felvtelt a prizsi Conservatoire-ba Cherubini, az intzmny vezetje - gtolta meg, ezrt Bcsben maradt, ahol zeneelmletet s ellenpontot - tanult magnton, kzben hangversenykrtra indult. Londoni - bemutatkozst szmos jabb meghvs kvette Franciaorszgba, Svjcba, - majd ismt Angliba. Msorn tbbnyire a npszer operkra rt - brndok, parafrzisok szerepeltek. 1827-ben egy idre a francia - fvrosban telepedett le, ahol tehetsgnek ksznheten a trsasgi - let nnepelt hse lett. Itt ismerkedett meg Chopinnal, akitl - eltanulta a zongorajtk rnyalatait, valamint Berliozzal, akitl a - zenekari nyelvet s a programzene j vvmnyait sajttotta el. Az - 1830-as vek vgn jabb hangversenykrutakra indult, bejrta egsz - Eurpt Portuglitl Oroszorszgig. 1835-ben a genfi konzervatrium - tanra lett. Az 1838-as pesti rvzkatasztrfa krosultjait Bcsben - rendezett hangversenyeinek jvedelmvel tmogatta. Ugyanilyen bkezen - jrult hozz nhny vvel ksbb a bonni Beethoven-emlkm - ltrehozshoz. Az oroszorszgi hangversenykrutakon ismerte meg az - orosz komponistk mvszett, valamint szerelmt, Carolyne zu - Sayn-Wittgenstein hercegnt, aki a ksbb irodalmi munkssgban is - kzremkdtt. A negyvenes vek elejn Weimarban az udvari sznhz - karmestere lett: npszerstette s veznyelte kortrsai mveit, - tantotta a kontinens minden sarkbl rkez plyakezd muzsikusokat s - komponistkat. Liszt lete utols vtizedeit Weimar, Rma s Budapest - kztt osztotta meg. A budapesti Zeneakadmia els veiben kszsggel - segtette a magyar zenekultra jonnan kialakul kzpontjt, s az v - nhny hnapjban tantott is itt. - - Tovbb a szcikkhez - __________________________________________________________________ - - A kezdlapon legutbb megjelent szcikkek: I. Kroly magyar kirly - Aerosmith Giuseppe Verdi Harisnya Kznsges cserepestekns - Nrnbergi per Bartk Bla - - Tovbbi kiemelt lapok Kiemelt szcikk RSS - - - A nap kpe - - Kznsges csimpnz (Pan troglodytes) - - Kznsges csimpnz (Pan troglodytes) - - Teljes mret Archvum Kiemelt kp RSS - -Tudtad? - - Tudtad-e, hogy - * a msodik vilghbor utni Budapesten egy villamoskocsiban - knyvtrat mkdtettek? - * a knai csszrokat, majd Mao Ce-tungot is a - vanszuj-kszntssel dvzltk, aminek jelentse tzezer v, - s amibl a japn banzj sz is ered? - * a dzsihd t jelentse kzl csak az egyik a szent hbor s nem - is a legfontosabb, az elsdleges a szv/llek dzsihdja (dzsihd - bin nafsz/kalb): bels kzdelem a gonoszsg ellen? - * Kanada neve a Szent Lrinc-folynl beszlt irokz kanata szbl - ered, melynek jelentse falu, telepls? - - - Aktulis - - Az Emlkm - Az Emlkm - Transtrmer - Transtrmer - Dan Shechtman - Dan Shechtman - Steve Jobs - Steve Jobs - * Oktber 21-n Szojuz raktval plyra lltottk az eurpai - Galileo navigcis rendszer els kt mholdjt. - * Oktber 16-n felavattk Martin Luther King Nobel-bkedjas - polgrjogi harcos 9 mteres szobrt is magba foglal emlmvt - Washington szvben a National Mall nemzeti Emlkparkban George - Washington s Abraham Lincoln emlkmve mellett. - * Oktber 15-n felavattk Boldog II. Jnos Pl els szobrt - Oroszorszgban, melyet a moszkvai nemzetkzi irodalmi knyvtr - udvarn lltottak fel. - * Oktber 11-n elhunyt Mszly Dezs Kossuth-djas magyar r, - klt, mfordt, dramaturg, Villon s Shakespeare mveinek avatott - tolmcsolja. - * Oktber 10-n, a Nobel-djas ht utols napjn a 2011-es - kzgazdasgi Nobel-emlkdjat Thomas J. Sargent s Christopher A. - Sims rszre tltk oda a gazdasgi folyamatokban az okok s - hatsok klnvlasztsrl szl elmleteikrt. - * Oktber 7-n, a Nobel-djas ht tdik napjn a 2011-es - Nobel-bkedjat Ellen Johnson-Sirleaf libriai elnk, Leymah Gbowee - libriai bkeaktivista s Tavakkul Karmn jemeni jogvd rszre - tltk oda megosztva a nk jogainak vdelmrt jelentettk be az - osli Norvg Nobel Intzetben. - * Oktber 6-n, a Nobel-djas ht negyedik napjn az idei irodalmi - Nobel-djat Tomas Transtrmer svd r, klt, mfordt rszre - tltk oda pontos, letisztult klti kpeirt, amelyek rvn j - fnyben lthatjuk a valsgot. - * Oktber 5-n, a Nobel-djas ht harmadik napjn az idei kmiai - Nobel-djat Daniel Shechtman kutat rszre tltk oda a - kvzikristlyok felfedezsrt. - * Oktber 5-n elhunyt Steve Jobs, az Apple Inc. elnke, korbbi - vezrigazgatja, az informatikai s a szrakoztat ipar kiemelked - alakja. - * Oktber 4-n, a Nobel-djas ht msodik napjn az idei fizikai - Nobel-djat Saul Perlmutter, Brian P. Schmidt s Adam G. Riess - csillagszoknak megosztva tltk oda a vilgegyetem gyorsul - temben val tgulsnak tvoli szupernvk megfigyelsvel trtnt - felfedezsrt. - * Oktber 3-n megkezddtt a 2011. vben Nobel-djjal kitntettek - nvsornak kihirdetse; az lettani-orvosi Nobel-dj felt Ralph - Steinman rszre tltk oda a dendritikus sejtekkel kapcsolatos - munkssgrt (a djazott a kihirdets eltt nhny nappal hunyt - el), msik felt megosztva veheti t december 10-n Bruce A. - Beutler s Jules Hoffmann az gynevezett termszetes vagy - veleszletett immunrendszer kutatsrt. - __________________________________________________________________ - - Hallesetek a kzelmltban - - - Ezen a napon - - Ma 2011. oktber 21. van, - * Orsolya nvnapja - - Nobel Alfrd - Nobel Alfrd - Tra-tekercs - Tra-tekercs - * Alfred Nobel nnepnapja - * Fldnkrt vilgnap (Magyarorszgon 1992 ta) - * Szent Orsolya s vrtantrsainak emlknapja - * Szimchat Tr (a Tra rmnnepe, napnyugtig) - * Boldog IV. Kroly kirly emlknapja (2004 ta) - * almanap, az almafk s a gymlcssk napja az Egyeslt - Kirlysgban - * a nachos nemzetkzi napja Mexikban s az Amerikai Egyeslt - llamokban - * polk napja Thaifldn - * tengerentli knaiak napja Tajvanon - * Trafalgar-nap az Egyeslt Kirlysgban - * Dizzy Gillespie szletsnapja - - vfordulk - * 247 ve, 1764-ben szletett Bihari Jnos cigny szrmazs magyar - zeneszerz s hegedmvsz, a Rkczi-indul vlelmezett szerzje, - akinek virtuz jtkt Liszt Ferenc is hallgatta s nagy - elismerssel emlkezett meg rla ( 1827). - * 150 ve, 1861-ben szletett Klupathy Jen fizikus, az MTA tagja, a - gyakorlati fizika kiemelked alakja( 1931). - * 140 ve, 1871-ben szletett Plfy Mric geolgus, hidrogeolgus, - egyetemi magntanr; a budapesti Fldtani Intzet fgeolgusa, a - Magyarhoni Fldtani Trsulat titkra, majd elnke, az MTA tagja - ( 1930). - * 100 ve, 1911-ben kttt hzassgot HabsburgLotaringiai Kroly - fherceg s Zita Bourbonprmai hercegn (1916-tl IV. Kroly - magyar kirly s Zita magyar kirlyn, illetve I. Kroly osztrk - csszr s Zita osztrk csszrn). - * 80 ve, 1931-ben hunyt el Arthur Schnitzler osztrk r, drmar - (* 1862) . - * 70 ve, 1941-ben szletett Jankovics Marcell Kossuth-djas - rajzfilmrendez, knyvillusztrtor, kultrtrtnsz, - kultrpolitikus. - - Jankovics Marcell - Jankovics Marcell - Arthur Schnitzler - Arthur Schnitzler - Hzassgkts - Hzassgkts - Bihari Jnos - Bihari Jnos - - Tovbbi nnepek nnepek RSS Tovbbi vfordulk vfordulk RSS - - - Ismerkeds a Wikipdival - - * dvzllap Bemutatkozik a Wikipdia. - * Els lpsek Didaktikusan felptett bevezet a Wikipdia - szerkesztsbe. - * Segtsg Egyszer, kzepes s halad szerkesztsi tippek, - tmutatk. - * Wikifogalmak Minden, ami elsre knaiul hangzik a Wikipdin, itt - magyarzatra lel. - * Kocsmafal kezdknek Az j szerkeszt krdez, aki tud, vlaszol. - * Homokoz jdonslt szerkesztink szabadon garzdlkodhatnak - benne. - * Mentorls Kezdeti lpseidhez tmogatst kaphatsz egy - tapasztaltabb szerkeszttl. - - Szerkeszti kzssg - - * Kzssgi portl A magyar Wikipdia szerkeszti kzssgnek - virtulis dolgozszobja. - * Kapcsolatfelvtel / Contact Kapcsolatfelvteli lehetsgek - ttekintse. - * Nagykvetsg / Embassy Kapcsolat ms nyelv Wikipdikkal. - * zenfal A Wikipdia klnbz tisztsgviselihez intzhet - krsek s krdsek platformja. - * Adminisztrtorok zenfala Az adminisztrtori beavatkozst - ignyl rendkvli esetekre. - * Wikipdia-IRC Instant csevej, rtekezlet lben a tbbi - szerkesztvel (belps). - * Esemnyek - A Wikipdia letben soron kvetkez esemnyek. - - - Wikipdia ms nyelveken - - Ez a magyar nyelv Wikipdia. 2003-ban indult, jelenleg 202 963 - szcikket tartalmaz. A Wikipdia sok ms nyelven is elrhet. - * A legnagyobb Wikipdik (tbb mint 500 000 szcikkel): angol, - francia, holland, japn, lengyel, nmet, olasz, orosz, portugl, - spanyol. - - * Wikipdik a Magyarorszgon hivatalosan elismert nemzetisgek - nyelvein: bolgr, cigny, grg, horvt, lengyel, nmet, rmny, - romn, ruszin, szerb, szerbhorvt, szlovk, szlovn, ukrn. - - Tovbbi nyelvek (magyarul) Tovbbi nyelvek (angolul, legfrissebb) - -Wikimdia-trslapok - - A Wikipdit a nonprofit Wikimdia Alaptvny zemelteti. A Wikimdia - szmos tbbnyelv s nylt tartalm trslapot zemeltet: - - Wikisztr Wikisztr - Tbbnyelv sztr s szinonimasztr Wikidzet Wikidzet - Tbbnyelv idzet- s szlsgyjtemny Wikifajok Wikifajok - Rendszertani adatbzis Wikiknyvek Wikiknyvek - Szabad kziknyvek s tmutatk - Wikimedia Commons Wikimedia Commons - Szabad mdiallomnyok gyjtemnye Wikiegyetem Wikiegyetem - Jegyzetek s tanulsi segdletek Wikiforrs Wikiforrs - Szabad forrsmunkk Meta-Wiki Meta-Wiki - A Wikimedia projektek koordincija - - Felhasznlsi felttelek - - A Wikipdiban tallhat szvegekre s egyes kpekre a Creative Commons - Nevezd meg! gy add tovbb! 3.0 (CC-BY-SA-3.0) licenc vonatkozik. - Cc.logo.circle.svg - Cc-by new.svg - Cc-sa.svg - * Minden szerkesztsed ezen licenc elfogadst s alkalmazst - jelenti, mellyel hozzjrulsz, hogy a mvet brki mdosthatja, azt - brmilyen clra felhasznlhatja. - * A Wikipdia tartalmnak jrafelhasznlsakor (tdolgozs esetn is) - meg kell adnod, hogy a mre a fenti licenc vonatkozik, tovbb meg - kell nevezned a forrsul szolgl szcikket; internetes kzzttel - esetn a forrsmre mutat linket kell elhelyezned honlapodon. - * A kpek felhasznlsi feltteleit kln kell ellenrizned. - - Felhasznlsi felttelek Jogi nyilatkozat jrafelhasznlsi - tmutat - A lap eredeti cme: - http://hu.wikipedia.org/w/index.php?title=Kezd%C5%91lap&oldid=9469813 - Kategria: - * Hibs fjlhivatkozsokat tartalmaz lapok - - Rejtett kategria: - * Kezdlap - - Mit gondolsz errl az oldalrl? - - Arra krnk, sznj egy percet a cikk rtkelsre! A visszajelzsek - segtenek az oldal fejlesztsben. - Megbzhatsg: [(bizonytalan)] Teljessg: [(bizonytalan)] - Trgyilagossg: [(bizonytalan)] Stlus: [(bizonytalan)] Klds - ____________ - -Szemlyes eszkzk - - * Bejelentkezs / fik ltrehozsa - -Nvterek - - * Kezdlap - * Vitalap - -Vltozk - -Nzetek - - * Olvass - * Lapforrs - * Laptrtnet - -Mveletek - -Keress - - ____________________ (Submit) Keress - -Navigci - - * Kezdlap - * Tartalom - * Kiemeltek - * Friss vltoztatsok - * Lap tallomra - * Tudakoz - -Rszvtel - - * Kezdknek - * Segtsg - * Kzssgi portl - * Kapcsolatfelvtel - * Adomnyok - -Nyomtats/exportls - - * Knyv ksztse - * Letlts mint PDF - * Nyomtathat vltozat - -Eszkzk - - * Mi hivatkozik erre - * Kapcsold vltoztatsok - * Specilis lapok - * Link erre a vltozatra - * Hogyan hivatkozz erre a lapra - * Lap rtkelse - -Ms nyelveken - - * English - * - * Azrbaycanca - * Armneashce - * - * Catal - * esky - * Dansk - * Deutsch - * Eesti - * - * Simple English - * Espaol - * Esperanto - * Euskara - * - * Franais - * Galego - * - * - * - * Hrvatski - * Bahasa Indonesia - * Italiano - * - * Kreyl ayisyen - * Latina - * Lietuvi - * Bahasa Melayu - * Nederlands - * - * - * Norsk (bokml) - * Norsk (nynorsk) - * Polski - * Portugus - * Romn - * - * Slovenina - * Slovenina - * / Srpski - * Suomi - * Svenska - * Tagalog - * - * Trke - * - * Ting Vit - * Volapk - * Winaray - * - - * A lap utols mdostsa: 2011. mrcius 31., 21:44 - * A lap szvege Creative Commons Nevezd meg! gy add tovbb! 3.0 - licenc alatt van; egyes esetekben ms mdon is felhasznlhat. - Rszletekrt lsd a felhasznlsi feltteleket. - - * Adatvdelmi irnyelvek - * A Wikipdirl - * Jogi nyilatkozat - * Mobile view - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/hu/UTF-8/wikitop_hu_UTF-8.txt b/src/tests/testdata/hu/UTF-8/wikitop_hu_UTF-8.txt deleted file mode 100644 index 01ed205..0000000 --- a/src/tests/testdata/hu/UTF-8/wikitop_hu_UTF-8.txt +++ /dev/null @@ -1,388 +0,0 @@ -Kezdőlap - - A Wikipédiából, a szabad enciklopédiából - Ez a lap egy ellenőrzött változata részletek megjelenítése/elrejtése - Ez a közzétett változat, ellenőrizve: 2011. március 31. - - Pontosság ellenőrzött - - Ugrás: navigáció, keresés - - Üdvözlünk a Wikipédiában! - Ezt az enciklopédiát az olvasói szerkesztik - A magyar változatnak 202 963 szócikke van, - ebből 623 kiemelt. - - * Társadalom - * Történelem - * Földrajz - - * Kultúra - * Művészet - * Sport - - * Természet - * Fizika - * Biológia - - * Matematika - * Technika - * Összes portál - - Mobilos változatKapcsolatfelvétel / Contact - - Kiemelt cikk - - Liszt-kaulbach.jpg - - Liszt Ferenc (németül Franz Liszt) (Doborján, 1811. október 22. – - Bayreuth, 1886. július 31.) a 19. század egyik legjelentősebb - romantikus zeneszerzője, minden idők egyik legnagyobb zongoraművésze. - - Apja, Liszt Ádám, Esterházy herceg uradalmi hivatalnoka, korán - felismerte fiának kivételes zenei tehetségét, és minden lehetőséget - megragadott kibontakoztatására. Liszt kilencéves korában már - nyilvánosság előtt zongorázott Sopronban és Pozsonyban, majd műpártoló - főurak támogatásával hamarosan Bécsben folytathatta tanulmányait Czerny - és Salieri tanítványaként. 1822. december 1-jén mutatkozott be az - osztrák fővárosban. Első, nagy feltűnést keltő bécsi koncertjén az - akkor 11 éves művészre még Beethoven is felfigyelt. Itt jelent meg - nyomtatásban első műve, variációi ugyanarra a Diabelli-témára - (keringőre), amelyre Beethoven híres zongoraművét írta. Az ifjú Liszt - bécsi szereplése, majd négy nagy sikerű pesti, illetve budai koncertje - után 1823 őszén atyjával az európai művészeti élet központjába, - Párizsba utazott. - - Felvételét a párizsi Conservatoire-ba Cherubini, az intézmény vezetője - gátolta meg, ezért Bécsben maradt, ahol zeneelméletet és ellenpontot - tanult magánúton, közben hangversenykörútra indult. Londoni - bemutatkozását számos újabb meghívás követte Franciaországba, Svájcba, - majd ismét Angliába. Műsorán többnyire a népszerű operákra írt - ábrándok, parafrázisok szerepeltek. 1827-ben egy időre a francia - fővárosban telepedett le, ahol tehetségének köszönhetően a társasági - élet ünnepelt hőse lett. Itt ismerkedett meg Chopinnal, akitől - eltanulta a zongorajáték árnyalatait, valamint Berliozzal, akitől a - zenekari nyelvet és a programzene új vívmányait sajátította el. Az - 1830-as évek végén újabb hangversenykörutakra indult, bejárta egész - Európát Portugáliától Oroszországig. 1835-ben a genfi konzervatórium - tanára lett. Az 1838-as pesti árvízkatasztrófa károsultjait Bécsben - rendezett hangversenyeinek jövedelmével támogatta. Ugyanilyen bőkezűen - járult hozzá néhány évvel később a bonni Beethoven-emlékmű - létrehozásához. Az oroszországi hangversenykörutakon ismerte meg az - orosz komponisták művészetét, valamint szerelmét, Carolyne zu - Sayn-Wittgenstein hercegnőt, aki a később irodalmi munkásságában is - közreműködött. A negyvenes évek elején Weimarban az udvari színház - karmestere lett: népszerűsítette és vezényelte kortársai műveit, - tanította a kontinens minden sarkából érkező pályakezdő muzsikusokat és - komponistákat. Liszt élete utolsó évtizedeit Weimar, Róma és Budapest - között osztotta meg. A budapesti Zeneakadémia első éveiben készséggel - segítette a magyar zenekultúra újonnan kialakuló központját, és az év - néhány hónapjában tanított is itt. - - Tovább a szócikkhez - __________________________________________________________________ - - A kezdőlapon legutóbb megjelent szócikkek: I. Károly magyar király • - Aerosmith • Giuseppe Verdi • Harisnya • Közönséges cserepesteknős • - Nürnbergi per • Bartók Béla - - További kiemelt lapok Kiemelt szócikk RSS - - - A nap képe - - Közönséges csimpánz (Pan troglodytes) - - Közönséges csimpánz (Pan troglodytes) - - Teljes méret • Archívum Kiemelt kép RSS - -Tudtad? - - Tudtad-e, hogy… - * …a második világháború utáni Budapesten egy villamoskocsiban - könyvtárat működtettek? - * …a kínai császárokat, majd Mao Ce-tungot is a - „vanszuj”-köszöntéssel üdvözölték, aminek jelentése „tízezer év”, - és amiből a japán banzáj szó is ered? - * …a dzsihád öt jelentése közül csak az egyik a „szent háború” és nem - is a legfontosabb, az elsődleges a „szív/lélek” dzsihádja (dzsihád - bin nafsz/kalb): „belsõ küzdelem a gonoszság ellen”? - * …Kanada neve a Szent Lőrinc-folyónál beszélt irokéz kanata szóból - ered, melynek jelentése „falu”, „település”? - - - Aktuális - - Az Emlékmű - Az Emlékmű - Tranströmer - Tranströmer - Dan Shechtman - Dan Shechtman - Steve Jobs - Steve Jobs - * Október 21-én Szojuz rakétával pályára állították az európai - Galileo navigációs rendszer első két műholdját. - * Október 16-án felavatták Martin Luther King Nobel-békedíjas - polgárjogi harcos 9 méteres szobrát is magába foglaló emléművét - Washington szívében a National Mall nemzeti Emlékparkban George - Washington és Abraham Lincoln emlékműve mellett. - * Október 15-én felavatták Boldog II. János Pál első szobrát - Oroszországban, melyet a moszkvai nemzetközi irodalmi könyvtár - udvarán állítottak fel. - * Október 11-én elhunyt Mészöly Dezső Kossuth-díjas magyar író, - költő, műfordító, dramaturg, Villon és Shakespeare műveinek avatott - tolmácsolója. - * Október 10-én, a Nobel-díjas hét utolsó napján a 2011-es - közgazdasági Nobel-emlékdíjat Thomas J. Sargent és Christopher A. - Sims részére ítélték oda a gazdasági folyamatokban az okok és - hatások különválasztásáról szóló elméleteikért. - * Október 7-én, a Nobel-díjas hét ötödik napján a 2011-es - Nobel-békedíjat Ellen Johnson-Sirleaf libériai elnök, Leymah Gbowee - libériai békeaktivista és Tavakkul Karmán jemeni jogvédő részére - ítélték oda megosztva a nők jogainak védelméért – jelentették be az - oslói Norvég Nobel Intézetben. - * Október 6-án, a Nobel-díjas hét negyedik napján az idei irodalmi - Nobel-díjat Tomas Tranströmer svéd író, költő, műfordító részére - ítélték oda „pontos, letisztult költői képeiért, amelyek révén új - fényben láthatjuk a valóságot”. - * Október 5-én, a Nobel-díjas hét harmadik napján az idei kémiai - Nobel-díjat Daniel Shechtman kutató részére ítélték oda a - kvázikristályok felfedezéséért. - * Október 5-én elhunyt Steve Jobs, az Apple Inc. elnöke, korábbi - vezérigazgatója, az informatikai és a szórakoztató ipar kiemelkedő - alakja. - * Október 4-én, a Nobel-díjas hét második napján az idei fizikai - Nobel-díjat Saul Perlmutter, Brian P. Schmidt és Adam G. Riess - csillagászoknak megosztva ítélték oda a világegyetem gyorsuló - ütemben való tágulásának távoli szupernóvák megfigyelésével történt - felfedezéséért. - * Október 3-án megkezdődött a 2011. évben Nobel-díjjal kitüntettek - névsorának kihirdetése; az élettani-orvosi Nobel-díj felét Ralph - Steinman részére ítélték oda a dendritikus sejtekkel kapcsolatos - munkásságáért (a díjazott a kihirdetés előtt néhány nappal hunyt - el), másik felét megosztva veheti át december 10-én Bruce A. - Beutler és Jules Hoffmann az úgynevezett természetes vagy - veleszületett immunrendszer kutatásáért. - __________________________________________________________________ - - Halálesetek a közelmúltban - - - Ezen a napon - - Ma 2011. október 21. van, - * Orsolya névnapja - - Nobel Alfréd - Nobel Alfréd - Tóra-tekercs - Tóra-tekercs - * Alfred Nobel ünnepnapja - * Földünkért világnap (Magyarországon 1992 óta) - * Szent Orsolya és vértanútársainak emléknapja - * Szimchat Tórá (a Tóra örömünnepe, napnyugtáig) - * Boldog IV. Károly király emléknapja (2004 óta) - * almanap, az almafák és a gyümölcsösök napja az Egyesült - Királyságban - * a nachos nemzetközi napja Mexikóban és az Amerikai Egyesült - Államokban - * ápolók napja Thaiföldön - * tengerentúli kínaiak napja Tajvanon - * Trafalgar-nap az Egyesült Királyságban - * Dizzy Gillespie születésnapja - - Évfordulók - * 247 éve, 1764-ben született Bihari János cigány származású magyar - zeneszerző és hegedűművész, a Rákóczi-induló vélelmezett szerzője, - akinek virtuóz játékát Liszt Ferenc is hallgatta és nagy - elismeréssel emlékezett meg róla († 1827). - * 150 éve, 1861-ben született Klupathy Jenő fizikus, az MTA tagja, a - gyakorlati fizika kiemelkedő alakja(† 1931). - * 140 éve, 1871-ben született Pálfy Móric geológus, hidrogeológus, - egyetemi magántanár; a budapesti Földtani Intézet főgeológusa, a - Magyarhoni Földtani Társulat titkára, majd elnöke, az MTA tagja - († 1930). - * 100 éve, 1911-ben kötött házasságot Habsburg–Lotaringiai Károly - főherceg és Zita Bourbon–pármai hercegnő (1916-tól IV. Károly - magyar király és Zita magyar királyné, illetve I. Károly osztrák - császár és Zita osztrák császárné). - * 80 éve, 1931-ben hunyt el Arthur Schnitzler osztrák író, drámaíró - (* 1862) . - * 70 éve, 1941-ben született Jankovics Marcell Kossuth-díjas - rajzfilmrendező, könyvillusztrátor, kultúrtörténész, - kultúrpolitikus. - - Jankovics Marcell - Jankovics Marcell - Arthur Schnitzler - Arthur Schnitzler - Házasságkötés - Házasságkötés - Bihari János - Bihari János - - További ünnepek Ünnepek RSS További évfordulók Évfordulók RSS - - - Ismerkedés a Wikipédiával - - * Üdvözlőlap – Bemutatkozik a Wikipédia. - * Első lépések – Didaktikusan felépített bevezető a Wikipédia - szerkesztésébe. - * Segítség – Egyszerű, közepes és haladó szerkesztési tippek, - útmutatók. - * Wikifogalmak – Minden, ami elsőre kínaiul hangzik a Wikipédián, itt - magyarázatra lel. - * Kocsmafal kezdőknek – Az új szerkesztő kérdez, aki tud, válaszol. - * Homokozó – Újdonsült szerkesztőink szabadon garázdálkodhatnak - benne. - * Mentorálás – Kezdeti lépéseidhez támogatást kaphatsz egy - tapasztaltabb szerkesztőtől. - - Szerkesztői közösség - - * Közösségi portál – A magyar Wikipédia szerkesztői közösségének - virtuális dolgozószobája. - * Kapcsolatfelvétel / Contact – Kapcsolatfelvételi lehetőségek - áttekintése. - * Nagykövetség / Embassy – Kapcsolat más nyelvű Wikipédiákkal. - * Üzenőfal – A Wikipédia különböző tisztségviselőihez intézhető - kérések és kérdések platformja. - * Adminisztrátorok üzenőfala – Az adminisztrátori beavatkozást - igénylő rendkívüli esetekre. - * Wikipédia-IRC – Instant csevej, értekezlet élőben a többi - szerkesztővel (belépés). - * Események - A Wikipédia életében soron következő események. - - - Wikipédia más nyelveken - - Ez a magyar nyelvű Wikipédia. 2003-ban indult, jelenleg 202 963 - szócikket tartalmaz. A Wikipédia sok más nyelven is elérhető. - * A legnagyobb Wikipédiák (több mint 500 000 szócikkel): angol, - francia, holland, japán, lengyel, német, olasz, orosz, portugál, - spanyol. - - * Wikipédiák a Magyarországon hivatalosan elismert nemzetiségek - nyelvein: bolgár, cigány, görög, horvát, lengyel, német, örmény, - román, ruszin, szerb, szerbhorvát, szlovák, szlovén, ukrán. - - További nyelvek (magyarul) • További nyelvek (angolul, legfrissebb) - -Wikimédia-társlapok - - A Wikipédiát a nonprofit Wikimédia Alapítvány üzemelteti. A Wikimédia - számos többnyelvű és nyílt tartalmú társlapot üzemeltet: - - Wikiszótár Wikiszótár - Többnyelvű szótár és szinonimaszótár Wikidézet Wikidézet - Többnyelvű idézet- és szólásgyűjtemény Wikifajok Wikifajok - Rendszertani adatbázis Wikikönyvek Wikikönyvek - Szabad kézikönyvek és útmutatók - Wikimedia Commons Wikimedia Commons - Szabad médiaállományok gyűjteménye Wikiegyetem Wikiegyetem - Jegyzetek és tanulási segédletek Wikiforrás Wikiforrás - Szabad forrásmunkák Meta-Wiki Meta-Wiki - A Wikimedia projektek koordinációja - - Felhasználási feltételek - - A Wikipédiában található szövegekre és egyes képekre a Creative Commons - Nevezd meg! – Így add tovább! 3.0 (CC-BY-SA-3.0) licenc vonatkozik. - Cc.logo.circle.svg - Cc-by new.svg - Cc-sa.svg - * Minden szerkesztésed ezen licenc elfogadását és alkalmazását - jelenti, mellyel hozzájárulsz, hogy a művet bárki módosíthatja, azt - bármilyen célra felhasználhatja. - * A Wikipédia tartalmának újrafelhasználásakor (átdolgozás esetén is) - meg kell adnod, hogy a műre a fenti licenc vonatkozik, továbbá meg - kell nevezned a forrásul szolgáló szócikket; internetes közzététel - esetén a forrásműre mutató linket kell elhelyezned honlapodon. - * A képek felhasználási feltételeit külön kell ellenőrizned. - - Felhasználási feltételek • Jogi nyilatkozat • Újrafelhasználási - útmutató - A lap eredeti címe: - „http://hu.wikipedia.org/w/index.php?title=Kezd%C5%91lap&oldid=9469813” - Kategória: - * Hibás fájlhivatkozásokat tartalmazó lapok - - Rejtett kategória: - * Kezdőlap - - Mit gondolsz erről az oldalról? - - Arra kérünk, szánj egy percet a cikk értékelésére! A visszajelzések - segítenek az oldal fejlesztésében. - Megbízhatóság: [(bizonytalan)] Teljesség: [(bizonytalan)] - Tárgyilagosság: [(bizonytalan)] Stílus: [(bizonytalan)] Küldés - ____________ - -Személyes eszközök - - * Bejelentkezés / fiók létrehozása - -Névterek - - * Kezdőlap - * Vitalap - -Változók - -Nézetek - - * Olvasás - * Lapforrás - * Laptörténet - -Műveletek - -Keresés - - ____________________ (Submit) Keresés - -Navigáció - - * Kezdőlap - * Tartalom - * Kiemeltek - * Friss változtatások - * Lap találomra - * Tudakozó - -Részvétel - - * Kezdőknek - * Segítség - * Közösségi portál - * Kapcsolatfelvétel - * Adományok - -Nyomtatás/exportálás - - * Könyv készítése - * Letöltés mint PDF - * Nyomtatható változat - -Eszközök - - * Mi hivatkozik erre - * Kapcsolódó változtatások - * Speciális lapok - * Link erre a változatra - * Hogyan hivatkozz erre a lapra - * Lap értékelése - -Más nyelveken - - * A lap utolsó módosítása: 2011. március 31., 21:44 - * A lap szövege Creative Commons Nevezd meg! – Így add tovább! 3.0 - licenc alatt van; egyes esetekben más módon is felhasználható. - Részletekért lásd a felhasználási feltételeket. - - * Adatvédelmi irányelvek - * A Wikipédiáról - * Jogi nyilatkozat diff --git a/src/tests/testdata/hu/iso-8859-2.txt b/src/tests/testdata/hu/iso-8859-2.txt new file mode 100644 index 0000000..cb0dc11 --- /dev/null +++ b/src/tests/testdata/hu/iso-8859-2.txt @@ -0,0 +1,3 @@ +Az ISO 8859-2 (hivatalosan ISO/IEC 8859-2, rviden s nem hivatalosan Latin-2) az ISO/IEC 8859-es karakterkdolsi szabvny msodik rsze. Az ISO ltal kettes szm latin bcnek nevezett 191 karakter mindegyiknek egybjtos (nyolcbites) kdjt adja meg. A 191 karakter kztt minden magyar kezetes bet megtallhat (a sok ms kszletbl hinyz s is). + +Az ISO_8859-2:1987 (mime rvidtsbl ismertebb nevn ISO-8859-2 (az "ISO" utn ktjellel)) az IANA-nak erre a szabvnyra pl karakterkszletnek neve, melyben a C0 (0x00-0x1F) s a C1 (0x80-0x9F) rsz az ISO/IEC 6429-ben meghatrozott vezrlkdokat tartalmazza. Az ISO/IEC 6429-ben s 2022-ben megadott escape szekvencikat nem hasznlja. Tovbbi ismert nevei: ISO_8859-2, latin2, l2 s csISOLatin2. diff --git a/src/tests/testdata/hu/windows-1250.txt b/src/tests/testdata/hu/windows-1250.txt new file mode 100644 index 0000000..ada5ffb --- /dev/null +++ b/src/tests/testdata/hu/windows-1250.txt @@ -0,0 +1 @@ +Jellemz r az els sztagra es hangsly (ebben a finnugor nyelvek s a szlovk nyelv hasonltanak hozz), a magnhangz-harmnia (barnulsotokrl zldlsetekrl), valamint a magnhangz-hosszsg s a hangsly egymstl fggetlen volta (amely szinte egyedliknt lehetv teszi az antik Idmrtkes versels alkalmazst). Hangrendszerre ezenkvl a lgy mssalhangzk (ny, ty, gy), az aspirlatlan zrhangok (h nlkl ejtett p, t, k, szemben pldul a germn nyelvekkel) s a palatlis magnhangzk eltti kemny mssalhangzk jelenlte jellemz (azaz lehetsges ne, ti stb. hangkapcsolat, nye, tyi helyett; szemben pldul az orosszal). Nincsenek benne valdi diftongusok (mint pldul a finnben vagy nmetben) s reduklt, vagyis elnyelt magnhangzk (mint pldul az angolban, nmetben). A specilis magyar a hang (mely a svdben s a perzsban is megvan) nehzsget okozhat a nyelvnket tanulknak. diff --git a/src/tests/testdata/it/UTF-8/wikitop_it_UTF-8.txt b/src/tests/testdata/it/UTF-8/wikitop_it_UTF-8.txt deleted file mode 100644 index d298589..0000000 --- a/src/tests/testdata/it/UTF-8/wikitop_it_UTF-8.txt +++ /dev/null @@ -1,352 +0,0 @@ -Pagina principale - - Da Wikipedia, l'enciclopedia libera. - Vai a: navigazione, cerca - - Benvenuti su Wikipedia - L'enciclopedia libera e collaborativa 851.600 voci in italiano - - Versione per dispositivi mobili - Sfoglia l'indice · Consulta il sommario · Naviga tra i portali tematici - - Vetrina - -Vetrina - - M44.jpg - - L'Ammasso del Presepe (Latino Præsepe, -mangiatoia-, noto anche come - Ammasso Alveare o con le sigle di catalogo M 44 o NGC 2632) è un - brillante ammasso aperto visibile nella costellazione del Cancro. È uno - dei più vicini al sistema solare e contiene una ricca popolazione di - stelle, più ampia di quelle di altri ammassi aperti vicini. In un cielo - nitido l'ammasso appare a occhio nudo come un oggetto nebuloso; - definito da Tolomeo "la massa nebulosa nel seno del Cancro", fu il - primo oggetto che Galileo osservò con il suo cannocchiale. - - Le sue componenti più luminose sono di magnitudine 6 e 7 e sono - perfettamente visibili con un binocolo anche dai centri urbani. L'età e - il moto proprio sono paragonabili a quelli delle Iadi, suggerendo che - entrambi gli ammassi abbiano un'origine comune; sia il Presepe che le - Iadi contengono inoltre delle giganti rosse e delle nane bianche, che - rappresentano gli ultimi stadi dell'evoluzione stellare, più un gran - numero di stelle di sequenza principale di classe A, F, G, K e M. - - Stime sulla sua distanza forniscono cifre che variano fra i 160 e i 187 - parsec, equivalenti a 520-610 anni luce, mentre sull'età, stimata sui - 580 milioni di anni, vi è maggiore accordo. - - Leggi la voce · Tutte le voci in vetrina · Voci di qualità - - Lo sapevi che... - -Lo sapevi che... - - Innocenzo Manzetti.PNG - - Innocenzo Manzetti (Aosta, 17 marzo 1826 – Aosta, 17 marzo 1877) è - stato uno scienziato ed inventore italiano. - Mente creativa ma pragmatica, nella sua epoca Innocenzo Manzetti era - noto sia nella comunità scientifica che nella sua città natale per le - sue invenzioni: un automa meccanico che suona il flauto, una automobile - a vapore, una pompa idraulica, uno speciale cemento idraulico, una - macchina per scolpire con eccezionale precisione, e altro ancora. - Secondo alcune fonti è stato addirittura il precursore dell'invenzione - del telefono, che studiò e perfezionò tra il 1844 e il 1864 pur non - brevettandolo - - Leggi la voce · Proponi un'altra voce - - Ricorrenze - -Ricorrenze - - Alfred Nobel - Alfred Nobel - - Nati il 21 ottobre... - Alfred Nobel (1833) - Edmondo De Amicis (1846) - Martin Gardner (1914) - - e morti... - Pietro Aretino (1556) - Jack Kerouac (1969) - François Truffaut (1984) - - In questo giorno accadde... - 1879 - Thomas Edison testa la prima lampadina funzionante - 1941 - Seconda guerra mondiale: Le truppe tedesche in Jugoslavia si - scatenano uccidendo migliaia di civili. - 1945 - In Francia le donne vanno a votare per la prima volta. - 1959 - A New York apre al pubblico il Guggenheim Museum progettato da - Frank Lloyd Wright. - 1994 - Corea del Nord e Stati Uniti firmano un accordo che obbliga la - Corea del Nord a fermare il suo programma di armamento nucleare. - - Ricorre oggi: Buddhismo, Giainismo, Induismo e Sikhismo celebrano - Diwali; la Chiesa cattolica celebra la memoria di sant'Ilarione e di - sant'Orsola. - - Nelle altre lingue - -Nelle altre lingue - - Di seguito sono elencate le 10 versioni maggiori di Wikipedia e una - selezione casuale di altre edizioni con un numero minore di voci: - - Le 10 maggiori (al 17 ottobre 2011): English (inglese) (3.768.027+) · - Deutsch (tedesco) (1.300.176+) · Français (francese) (1.161.858+) · - Italiano (851.600) · Polski (polacco) (836.970+) · Español (spagnolo) - (836.128+) · Русский (russo) (778.550+) · 日本語 (giapponese) (773.442+) · - Nederlands (olandese) (763.709+) · Português (portoghese) (701.185+) - - Dal mondo di Wikipedia: עברית (ebraico) · Lietuvių (lituano) · Norsk - nynorsk (norvegese nynorsk) · Íslenska (islandese) · Basa Jawa - (giavanese) · Asturianu (asturiano) · Winaray (waray-waray) · Lumbaart - (lombardo) · 贛語 (Gànyŭ) (gan) · مصرى (Maṣrī) (arabo d'Egitto) · Bikol - (bicolano centrale) · Oyghurque (uiguro) · Коми (komi) · ܠܫܢܐ ܐܪܡܝܐ - (assiro neo-aramaico) · Papiamentu (papiamento) · Dolnoserbšćina (basso - sorabo) · Мокшень кяль (mokša) · IsiZulu (zulu) · Fino' Chamoru - (chamorro) - - Oltre Wikipedia - -Oltre Wikipedia - - Wikipedia è gestita da Wikimedia Foundation, fondazione senza fini di - lucro, che supporta diversi altri progetti wiki dal contenuto libero e - multilingue: - - - Commons-logo.svg - Commons - Risorse multimediali condivise - - Wikiquote-logo.svg - Wikiquote - Raccolta di citazioni - - Wiktprintable without text.svg - Wikizionario - Dizionario e lessico - - Wikinews-logo.svg - Wikinotizie - Notizie a contenuto aperto - - Wikispecies-logo.svg - Wikispecies - Catalogo delle specie - - Wikibooks-logo.svg - Wikibooks - Manuali e libri di testo liberi scritti ex-novo - Wikisource-logo.svg - Wikisource - Opere già pubblicate e libere da copyright - Wikiversity-logo.svg - Wikiversità - Risorse e attività didattiche - Wikimedia Community Logo.svg - Meta-Wiki - Progetto di coordinamento Wikimedia - - Wikipedia - -Wikipedia - - Wikipedia è un'enciclopedia online, collaborativa e gratuita. - Disponibile in oltre 270 lingue, Wikipedia affronta sia gli argomenti - tipici delle enciclopedie tradizionali sia quelli presenti in - almanacchi, dizionari geografici e pubblicazioni specialistiche. - - Wikipedia è liberamente modificabile: chiunque può contribuire alle - voci esistenti o crearne di nuove. Ogni contenuto è pubblicato sotto - licenza Creative Commons CC-BY-SA e può pertanto essere copiato e - riutilizzato adottando la medesima licenza. - - Cosa posso fare? - -Cosa posso fare? - - * Vuoi partecipare? Leggi le regole fondamentali e le altre linee - guida; rispetta la wikiquette; la registrazione è consigliata ma - non obbligatoria. - * Vuoi provare? Modifica la pagina di prova o leggi come scrivere una - voce. - * Serve aiuto? Consulta le istruzioni o leggi dove fare una domanda. - * Cosa pensi di Wikipedia? Dacci il tuo parere. - - Novità da Wikipedia - -Novità da Wikipedia - - Voci nuove in evidenza: - - Isolotto di San Nicola · Siddhartha Mukherjee · Morris Motor - Company · Carrozze FS tipo TEE - - Sono nati i portali: - - Corea del Sud · Automobilismo · Aviazione · Arrampicata · Isole - - il Wikipediano, notizie dalla comunità - - Dagli altri progetti - -Dagli altri progetti - - Wikiquote-logo.svg Citazione del giorno da Wikiquote - - Cquote1.png Credo che non ci sia nessuno al mondo che non abbia una - lepre in pancia, ossia un ramo di follia: perché tutti siamo della - stessa pasta e dalle mie pere posso sapere quanto sian mature quelle - del vicino. Cquote2.png - Hans Grimmelshausen - - Commons-logo.svg Immagine del giorno da Commons - - Schimpanse, Pan troglodytes 3.JPG - - Scimpanzé comune (Pan troglodytes). - - Wikinews-logo.svg Ultime notizie da Wikinews - * Gheddafi è morto - * Consumo record di cocaina a Milano - * 7 studenti su 10 vittime di bullismo omofobico - * «Se l'Italia vuole il Ponte se lo deve pagare da sola» - * Rilasciato Gilad Shalit - * Bielorussia: vietato Gay Pride - - Wikisource-logo.svg Rilettura del mese su Wikisource - - Zibaldone di Giacomo Leopardi (XIX secolo) - - Zibaldone di pensieri I.djvu - - Sei ai tuoi primissimi passi su Wikisource? clicca qui! - - - Wikipedia non dà garanzie sulla validità dei contenuti - - Il progetto è ospitato dalla Wikimedia Foundation, che non può essere - ritenuta responsabile di eventuali errori contenuti in questo sito. - Ogni contributore è responsabile dei propri inserimenti. - Sala stampa · Contatti · FAQ · Copyright · Wikipedia sul tuo sito · - Citare Wikipedia · Donazioni · Not Italian? it-0? Click here - * Meta - * Wikizionario - * Wikibooks - * Wikinotizie - * Wikiquote - * Wikisource - * Wikiversità - * Wikispecies - * Commons - - Estratto da - "http://it.wikipedia.org/w/index.php?title=Pagina_principale&oldid=4406 - 3559" - Categoria: - * Pagina principale - -Strumenti personali - - * Entra / Registrati - -Namespace - - * Voce - * Discussione - -Varianti - -Visite - - * Leggi - * Visualizza sorgente - * Visualizza cronologia - -Azioni - -Ricerca - - ____________________ (Submit) Ricerca - -Navigazione - - * Pagina principale - * Ultime modifiche - * Una voce a caso - * Vetrina - * Aiuto - -Comunità - - * Portale Comunità - * Bar - * Il Wikipediano - * Fai una donazione - * Contatti - -Stampa/esporta - - * Crea un libro - * Scarica come PDF - * Versione stampabile - -Strumenti - - * Puntano qui - * Modifiche correlate - * Pagine speciali - * Link permanente - * Cita questa voce - -Altre lingue - - * English - * Deutsch - * Français - * Polski - * Español - * Русский - * 日本語 - * Nederlands - * Português - * Svenska - * 中文 - * Català - * Українська - * ‪Norsk (bokmål)‬ - * Suomi - * Tiếng Việt - * Česky - * Magyar - * 한국어 - * Türkçe - * Bahasa Indonesia - * Română - * فارسی - * العربية - * Dansk - * Esperanto - * Српски / Srpski - * Lietuvių - * Slovenčina - * עברית - - * Ultima modifica per la pagina: 00:23, 12 ott 2011. - * Il testo è disponibile secondo la licenza Creative Commons - Attribuzione-Condividi allo stesso modo; possono applicarsi - condizioni ulteriori. Vedi le condizioni d'uso per i dettagli. - Wikipedia® è un marchio registrato della Wikimedia Foundation, Inc. - - * Politica sulla privacy - * Informazioni su Wikipedia - * Avvertenze - * Mobile view - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/it/WINDOWS-1252/wikitop_it_WINDOWS-1252.txt b/src/tests/testdata/it/WINDOWS-1252/wikitop_it_WINDOWS-1252.txt deleted file mode 100644 index a1011b1..0000000 --- a/src/tests/testdata/it/WINDOWS-1252/wikitop_it_WINDOWS-1252.txt +++ /dev/null @@ -1,335 +0,0 @@ - L'Ammasso del Presepe (Latino Prsepe, -mangiatoia-, noto anche come - Ammasso Alveare o con le sigle di catalogo M 44 o NGC 2632) un - brillante ammasso aperto visibile nella costellazione del Cancro. uno - dei pi vicini al sistema solare e contiene una ricca popolazione di - stelle, pi ampia di quelle di altri ammassi aperti vicini. In un cielo - nitido l'ammasso appare a occhio nudo come un oggetto nebuloso; - definito da Tolomeo "la massa nebulosa nel seno del Cancro", fu il - primo oggetto che Galileo osserv con il suo cannocchiale. - - Le sue componenti pi luminose sono di magnitudine 6 e 7 e sono - perfettamente visibili con un binocolo anche dai centri urbani. L'et e - il moto proprio sono paragonabili a quelli delle Iadi, suggerendo che - entrambi gli ammassi abbiano un'origine comune; sia il Presepe che le - Iadi contengono inoltre delle giganti rosse e delle nane bianche, che - rappresentano gli ultimi stadi dell'evoluzione stellare, pi un gran - numero di stelle di sequenza principale di classe A, F, G, K e M. - - Stime sulla sua distanza forniscono cifre che variano fra i 160 e i 187 - parsec, equivalenti a 520-610 anni luce, mentre sull'et, stimata sui - 580 milioni di anni, vi maggiore accordo. - - Leggi la voce Tutte le voci in vetrina Voci di qualit - - Lo sapevi che... - -Lo sapevi che... - - Innocenzo Manzetti.PNG - - Innocenzo Manzetti (Aosta, 17 marzo 1826 Aosta, 17 marzo 1877) - stato uno scienziato ed inventore italiano. - Mente creativa ma pragmatica, nella sua epoca Innocenzo Manzetti era - noto sia nella comunit scientifica che nella sua citt natale per le - sue invenzioni: un automa meccanico che suona il flauto, una automobile - a vapore, una pompa idraulica, uno speciale cemento idraulico, una - macchina per scolpire con eccezionale precisione, e altro ancora. - Secondo alcune fonti stato addirittura il precursore dell'invenzione - del telefono, che studi e perfezion tra il 1844 e il 1864 pur non - brevettandolo - - Leggi la voce Proponi un'altra voce - - Ricorrenze - -Ricorrenze - - Alfred Nobel - Alfred Nobel - - Nati il 21 ottobre... - Alfred Nobel (1833) - Edmondo De Amicis (1846) - Martin Gardner (1914) - - e morti... - Pietro Aretino (1556) - Jack Kerouac (1969) - Franois Truffaut (1984) - - In questo giorno accadde... - 1879 - Thomas Edison testa la prima lampadina funzionante - 1941 - Seconda guerra mondiale: Le truppe tedesche in Jugoslavia si - scatenano uccidendo migliaia di civili. - 1945 - In Francia le donne vanno a votare per la prima volta. - 1959 - A New York apre al pubblico il Guggenheim Museum progettato da - Frank Lloyd Wright. - 1994 - Corea del Nord e Stati Uniti firmano un accordo che obbliga la - Corea del Nord a fermare il suo programma di armamento nucleare. - - Ricorre oggi: Buddhismo, Giainismo, Induismo e Sikhismo celebrano - Diwali; la Chiesa cattolica celebra la memoria di sant'Ilarione e di - sant'Orsola. - - Nelle altre lingue - -Nelle altre lingue - - Di seguito sono elencate le 10 versioni maggiori di Wikipedia e una - selezione casuale di altre edizioni con un numero minore di voci: - - Le 10 maggiori (al 17 ottobre 2011): English (inglese) (3.768.027+) - Deutsch (tedesco) (1.300.176+) Franais (francese) (1.161.858+) - Italiano (851.600) Polski (polacco) (836.970+) Espaol (spagnolo) - (836.128+) (russo) (778.550+) (giapponese) (773.442+) - Nederlands (olandese) (763.709+) Portugus (portoghese) (701.185+) - - Dal mondo di Wikipedia: (ebraico) Lietuvi (lituano) Norsk - nynorsk (norvegese nynorsk) slenska (islandese) Basa Jawa - (giavanese) Asturianu (asturiano) Winaray (waray-waray) Lumbaart - (lombardo) (Gny) (gan) (Mar) (arabo d'Egitto) Bikol - (bicolano centrale) Oyghurque (uiguro) (komi) - (assiro neo-aramaico) Papiamentu (papiamento) Dolnoserbina (basso - sorabo) (moka) IsiZulu (zulu) Fino' Chamoru - (chamorro) - - Oltre Wikipedia - -Oltre Wikipedia - - Wikipedia gestita da Wikimedia Foundation, fondazione senza fini di - lucro, che supporta diversi altri progetti wiki dal contenuto libero e - multilingue: - - - Commons-logo.svg - Commons - Risorse multimediali condivise - - Wikiquote-logo.svg - Wikiquote - Raccolta di citazioni - - Wiktprintable without text.svg - Wikizionario - Dizionario e lessico - - Wikinews-logo.svg - Wikinotizie - Notizie a contenuto aperto - - Wikispecies-logo.svg - Wikispecies - Catalogo delle specie - - Wikibooks-logo.svg - Wikibooks - Manuali e libri di testo liberi scritti ex-novo - Wikisource-logo.svg - Wikisource - Opere gi pubblicate e libere da copyright - Wikiversity-logo.svg - Wikiversit - Risorse e attivit didattiche - Wikimedia Community Logo.svg - Meta-Wiki - Progetto di coordinamento Wikimedia - - Wikipedia - -Wikipedia - - Wikipedia un'enciclopedia online, collaborativa e gratuita. - Disponibile in oltre 270 lingue, Wikipedia affronta sia gli argomenti - tipici delle enciclopedie tradizionali sia quelli presenti in - almanacchi, dizionari geografici e pubblicazioni specialistiche. - - Wikipedia liberamente modificabile: chiunque pu contribuire alle - voci esistenti o crearne di nuove. Ogni contenuto pubblicato sotto - licenza Creative Commons CC-BY-SA e pu pertanto essere copiato e - riutilizzato adottando la medesima licenza. - - Cosa posso fare? - -Cosa posso fare? - - * Vuoi partecipare? Leggi le regole fondamentali e le altre linee - guida; rispetta la wikiquette; la registrazione consigliata ma - non obbligatoria. - * Vuoi provare? Modifica la pagina di prova o leggi come scrivere una - voce. - * Serve aiuto? Consulta le istruzioni o leggi dove fare una domanda. - * Cosa pensi di Wikipedia? Dacci il tuo parere. - - Novit da Wikipedia - -Novit da Wikipedia - - Voci nuove in evidenza: - - Isolotto di San Nicola Siddhartha Mukherjee Morris Motor - Company Carrozze FS tipo TEE - - Sono nati i portali: - - Corea del Sud Automobilismo Aviazione Arrampicata Isole - - il Wikipediano, notizie dalla comunit - - Dagli altri progetti - -Dagli altri progetti - - Wikiquote-logo.svg Citazione del giorno da Wikiquote - - Cquote1.png Credo che non ci sia nessuno al mondo che non abbia una - lepre in pancia, ossia un ramo di follia: perch tutti siamo della - stessa pasta e dalle mie pere posso sapere quanto sian mature quelle - del vicino. Cquote2.png - Hans Grimmelshausen - - Commons-logo.svg Immagine del giorno da Commons - - Schimpanse, Pan troglodytes 3.JPG - - Scimpanz comune (Pan troglodytes). - - Wikinews-logo.svg Ultime notizie da Wikinews - * Gheddafi morto - * Consumo record di cocaina a Milano - * 7 studenti su 10 vittime di bullismo omofobico - * Se l'Italia vuole il Ponte se lo deve pagare da sola - * Rilasciato Gilad Shalit - * Bielorussia: vietato Gay Pride - - Wikisource-logo.svg Rilettura del mese su Wikisource - - Zibaldone di Giacomo Leopardi (XIX secolo) - - Zibaldone di pensieri I.djvu - - Sei ai tuoi primissimi passi su Wikisource? clicca qui! - - - Wikipedia non d garanzie sulla validit dei contenuti - - Il progetto ospitato dalla Wikimedia Foundation, che non pu essere - ritenuta responsabile di eventuali errori contenuti in questo sito. - Ogni contributore responsabile dei propri inserimenti. - Sala stampa Contatti FAQ Copyright Wikipedia sul tuo sito - Citare Wikipedia Donazioni Not Italian? it-0? Click here - * Meta - * Wikizionario - * Wikibooks - * Wikinotizie - * Wikiquote - * Wikisource - * Wikiversit - * Wikispecies - * Commons - - Estratto da - "http://it.wikipedia.org/w/index.php?title=Pagina_principale&oldid=4406 - 3559" - Categoria: - * Pagina principale - -Strumenti personali - - * Entra / Registrati - -Namespace - - * Voce - * Discussione - -Varianti - -Visite - - * Leggi - * Visualizza sorgente - * Visualizza cronologia - -Azioni - -Ricerca - - ____________________ (Submit) Ricerca - -Navigazione - - * Pagina principale - * Ultime modifiche - * Una voce a caso - * Vetrina - * Aiuto - -Comunit - - * Portale Comunit - * Bar - * Il Wikipediano - * Fai una donazione - * Contatti - -Stampa/esporta - - * Crea un libro - * Scarica come PDF - * Versione stampabile - -Strumenti - - * Puntano qui - * Modifiche correlate - * Pagine speciali - * Link permanente - * Cita questa voce - -Altre lingue - - * English - * Deutsch - * Franais - * Polski - * Espaol - * - * - * Nederlands - * Portugus - * Svenska - * - * Catal - * - * Norsk (bokml) - * Suomi - * Ting Vit - * esky - * Magyar - * - * Trke - * Bahasa Indonesia - * Romn - * - * - * Dansk - * Esperanto - * / Srpski - * Lietuvi - * Slovenina - * - - * Ultima modifica per la pagina: 00:23, 12 ott 2011. - * Il testo disponibile secondo la licenza Creative Commons - Attribuzione-Condividi allo stesso modo; possono applicarsi - condizioni ulteriori. Vedi le condizioni d'uso per i dettagli. - Wikipedia un marchio registrato della Wikimedia Foundation, Inc. - - * Politica sulla privacy - * Informazioni su Wikipedia - * Avvertenze - * Mobile view - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/it/iso-8859-1.txt b/src/tests/testdata/it/iso-8859-1.txt new file mode 100644 index 0000000..0afbc36 --- /dev/null +++ b/src/tests/testdata/it/iso-8859-1.txt @@ -0,0 +1,18 @@ +L'architettura longobarda costituita dall'insieme delle opere architettoniche +realizzate in Italia durante il regno dei Longobardi (568-774), con residuale +permanenza nell'Italia meridionale fino al X-XI secolo (Langobardia Minor), e +commissionate dai re e dai duchi longobardi. +L'attivit architettonica sviluppata in Langobardia Maior andata in gran parte +perduta, per lo pi a causa di successive ricostruzioni degli edifici sacri e +profani eretti tra VII e VIII secolo. A parte il Tempietto longobardo di +Cividale del Friuli, rimasto in gran parte intatto, gli edifici civili e +religiosi di Pavia, Monza o altre localit sono stati ampiamente rimaneggiati +nei secoli seguenti. Ancora integre rimangono cos soltanto poche architetture, +o perch inglobate negli ampliamenti successivi - come la chiesa di San +Salvatore a Brescia) -, o perch periferiche e di modeste dimensioni - come la +chiesa di Santa Maria foris portas a Castelseprio. Testimonianze maggiormente +fedeli alla forma originale si ritrovano, invece, nella Langobardia Minor: a +Benevento si conservano la chiesa di Santa Sofia, un ampio tratto delle Mura e +la Rocca dei Rettori, unici esempi superstiti di architettura militare +longobarda, mentre altre testimonianze si sono conservate in centri minori del +ducato beneventano e a Spoleto. diff --git a/src/tests/testdata/it/utf-8.txt b/src/tests/testdata/it/utf-8.txt new file mode 100644 index 0000000..cefd085 --- /dev/null +++ b/src/tests/testdata/it/utf-8.txt @@ -0,0 +1,18 @@ +L'architettura longobarda è costituita dall'insieme delle opere architettoniche +realizzate in Italia durante il regno dei Longobardi (568-774), con residuale +permanenza nell'Italia meridionale fino al X-XI secolo (Langobardia Minor), e +commissionate dai re e dai duchi longobardi. +L'attività architettonica sviluppata in Langobardia Maior è andata in gran parte +perduta, per lo più a causa di successive ricostruzioni degli edifici sacri e +profani eretti tra VII e VIII secolo. A parte il Tempietto longobardo di +Cividale del Friuli, rimasto in gran parte intatto, gli edifici civili e +religiosi di Pavia, Monza o altre località sono stati ampiamente rimaneggiati +nei secoli seguenti. Ancora integre rimangono così soltanto poche architetture, +o perché inglobate negli ampliamenti successivi - come la chiesa di San +Salvatore a Brescia) -, o perché periferiche e di modeste dimensioni - come la +chiesa di Santa Maria foris portas a Castelseprio. Testimonianze maggiormente +fedeli alla forma originale si ritrovano, invece, nella Langobardia Minor: a +Benevento si conservano la chiesa di Santa Sofia, un ampio tratto delle Mura e +la Rocca dei Rettori, unici esempi superstiti di architettura militare +longobarda, mentre altre testimonianze si sono conservate in centri minori del +ducato beneventano e a Spoleto. diff --git a/src/tests/testdata/ja/euc-jp.txt b/src/tests/testdata/ja/euc-jp.txt new file mode 100644 index 0000000..4bd0c16 --- /dev/null +++ b/src/tests/testdata/ja/euc-jp.txt @@ -0,0 +1,10 @@ +Extended Unix Code(EUC)ϡUNIXǤ褯Ȥʸɤ沽Ǥ롣 + + ܸEUC + JIS X 0208١ (EUC-JP) + JIS X 0213١ (EUC-JIS-2004) + ڹEUC (EUC-KR) + λEUC (EUC-CN) + λEUC (EUC-TW) + +ʤɤ롣 diff --git a/src/tests/testdata/ja/iso-2022-jp.txt b/src/tests/testdata/ja/iso-2022-jp.txt new file mode 100644 index 0000000..c0dc891 --- /dev/null +++ b/src/tests/testdata/ja/iso-2022-jp.txt @@ -0,0 +1,8 @@ +ISO/IEC 2022$B!J5l>N(B ISO 2022$B!K$O!"(B + + $BJ8;z=89g$r(B7$B%S%C%HId9f$^$?$O(B8$B%S%C%HId9f$GI=8=$9$k$?$a$N5;=Q!"$*$h$S(B + $BJ#?t$NJ8;z=89g$rC10l$NJ8;zId9f2=J}<0$K4^$a$k5;=Q(B + +$B$r5,Dj$9$k(BISO$B5,3J$G$"$k!#(BJIS$B$NBP1~5,3J$O(BJIS X 0202 $B!V>pJs5;=Q(B-$BJ8;zId9f$N9=B$5Z$S3HD%K!!W(B[1]$B!#(BEcma International$B$NBP1~5,3J$O(BECMA-35$B!#(B + +ISO/IEC 2022 $B$NId9f2=J}<0$O!"0lHL$K!"(B1$BJ8;z$K(B1$B%P%$%H$+(B2$B%P%$%H0J>e$r;H$&2DJQD9$NJ8;zId9f2=J}<0$G$"$k!#$$$/$D$+$NId9f2=I=8=$,(BISO/IEC 2022$B$N5!9=$r;H$C$F$$$k!#$?$H$($P!"(BISO-2022-JP$B$OF|K\8l$G9-$/;H$o$l$F$$$kId9f2=I=8=$G$"$j!"$$$o$f$k!V(BJIS$B%3!<%I!W$H$$$&$N$b$3$l$r;X$9$3$H$,0lHLE*$G$"$k!#(B diff --git a/src/tests/testdata/ja/shift_jis.txt b/src/tests/testdata/ja/shift_jis.txt new file mode 100644 index 0000000..a580281 --- /dev/null +++ b/src/tests/testdata/ja/shift_jis.txt @@ -0,0 +1 @@ +{{{{{{{{{{{{{{{{{{{ diff --git a/src/tests/testdata/ja/utf-16be.txt b/src/tests/testdata/ja/utf-16be.txt new file mode 100644 index 0000000..7703c2f Binary files /dev/null and b/src/tests/testdata/ja/utf-16be.txt differ diff --git a/src/tests/testdata/ja/utf-16le.txt b/src/tests/testdata/ja/utf-16le.txt new file mode 100644 index 0000000..b04c2f8 Binary files /dev/null and b/src/tests/testdata/ja/utf-16le.txt differ diff --git a/src/tests/testdata/ja/utf-8.txt b/src/tests/testdata/ja/utf-8.txt new file mode 100644 index 0000000..cd4b312 --- /dev/null +++ b/src/tests/testdata/ja/utf-8.txt @@ -0,0 +1,9 @@ +UTF-8（ユーティーエフはち、ユーティーエフエイト）はISO/IEC 10646 (UCS) とUnicodeで使える8ビット符号単位の文字符号化形式及び文字符号化スキーム。 + +正式名称は、ISO/IEC 10646では “UCS Transformation Format 8”、Unicodeでは “Unicode Transformation Format-8” という。両者はISO/IEC 10646とUnicodeのコード重複範囲で互換性がある。RFCにも仕様がある[1]。 + +2バイト目以降に「/」などのASCII文字が現れないように工夫されていることから、UTF-FSS (File System Safe) ともいわれる。旧名称はUTF-2。 + +データ交換方式、ファイル形式として、一般的にUTF-8は使われる傾向にある。 + +当初は、ベル研究所においてPlan 9で用いるエンコードとして、ロブ・パイクによる設計指針のもと、ケン・トンプソンによって考案された diff --git a/src/tests/testdata/ko/iso-2022-kr.txt b/src/tests/testdata/ko/iso-2022-kr.txt new file mode 100644 index 0000000..7ce57eb --- /dev/null +++ b/src/tests/testdata/ko/iso-2022-kr.txt @@ -0,0 +1,8 @@ +$)CISO/IEC 20224B + + 9.@Z A}GU@; 7:qF. :NH# 6G4B 8:qF. :NH#7N G%GvGO1b @'GQ 1b24B 0!:/ 9.@Z :NH#H- 9f=D@L8g GQ19>n@G 0f?l EUC-KR@L @L :NH#H- 9f=D@; ;g?kGQ4Y. diff --git a/src/tests/testdata/ko/uhc.smi b/src/tests/testdata/ko/uhc.smi new file mode 100644 index 0000000..1b71cb6 --- /dev/null +++ b/src/tests/testdata/ko/uhc.smi @@ -0,0 +1,16 @@ + + +EUC-KR.smi + + + +

EUC-KR +

EUC-KR KS X 1001 KS X 1003 ϴ 8Ʈ ڵ, EUC ̸ ǥ ѱ ϼ ڵ̱ ϼ̶ Ҹ. +

EUC-KR ڵ ȴ. +

128 Ʈ KS X 1003 Ѵ. +

128 ũų Ʈ KS X 1001 Ѵ. ڴ 128 ڵ尪 Ͽ 2Ʈ ǥȴ. +

KS X 1001 40-27 "" ڴ EUC-KR C0 A7 Ʈ ǥȴ. +

KS X 1001 ѱ ä ڸ Ͽ ԰ տ Ե ѱ ǥϴ Ȯ , κ EUC-KR ʰ CP949 ٸ Ͽ KS X 1001 ٱ ѱ ǥѴ. + + diff --git a/src/tests/testdata/ko/utf-16.le b/src/tests/testdata/ko/utf-16.le new file mode 100644 index 0000000..8fe2e10 Binary files /dev/null and b/src/tests/testdata/ko/utf-16.le differ diff --git a/src/tests/testdata/ko/utf-32.be b/src/tests/testdata/ko/utf-32.be new file mode 100644 index 0000000..37162f5 Binary files /dev/null and b/src/tests/testdata/ko/utf-32.be differ diff --git a/src/tests/testdata/ko/utf-8.txt b/src/tests/testdata/ko/utf-8.txt new file mode 100644 index 0000000..d3ec5d4 --- /dev/null +++ b/src/tests/testdata/ko/utf-8.txt @@ -0,0 +1,3 @@ +UTF-8은 유니코드를 위한 가변 길이 문자 인코딩 방식 중 하나로, 켄 톰프슨과 롭 파이크가 만들었다. 본래는 FSS-UTF(File System Safe UCS/Unicode Transformation Format)라는 이름으로 제안되었다. + +UTF-8 인코딩은 유니코드 한 문자를 나타내기 위해 1바이트에서 4바이트까지를 사용한다. 예를 들어서, U+0000부터 U+007F 범위에 있는 ASCII 문자들은 UTF-8에서 1바이트만으로 표시된다. 4바이트로 표현되는 문자는 모두 기본 다국어 평면(BMP) 바깥의 유니코드 문자이며, 거의 사용되지 않는다. UTF-16과 UTF-8 중 어느 인코딩이 더 적은 바이트를 사용하는지는 문자열에서 사용된 코드 포인트에 따라 달라지며, 실제로 DEFLATE와 같은 일반적인 압축 알고리즘을 사용할 경우 이 차이는 무시할 수 있을 정도이다. 이러한 압축 알고리즘을 사용하기 힘들고 크기가 중요할 경우 유니코드 표준 압축 방식을 대신 사용할 수 있다. diff --git a/src/tests/testdata/lt/iso-8859-10.txt b/src/tests/testdata/lt/iso-8859-10.txt new file mode 100644 index 0000000..d005822 --- /dev/null +++ b/src/tests/testdata/lt/iso-8859-10.txt @@ -0,0 +1,3 @@ +Vincentas van Gogas (ol. Vincent van Gogh, 1853 m. kovo 30 d. Grot Zunderte, +Nyderlandai - 1890 m. liepos 29 d. Overe prie Uazos, Pranczija) - oland +tapytojas ir grafikas, postimpresionistas. diff --git a/src/tests/testdata/lt/iso-8859-13.txt b/src/tests/testdata/lt/iso-8859-13.txt new file mode 100644 index 0000000..1e3b02e --- /dev/null +++ b/src/tests/testdata/lt/iso-8859-13.txt @@ -0,0 +1,3 @@ +Vincentas van Gogas (ol. Vincent van Gogh, 1853 m. kovo 30 d. Grot Zunderte, +Nyderlandai - 1890 m. liepos 29 d. Overe prie Uazos, Pranczija) - oland +tapytojas ir grafikas, postimpresionistas. diff --git a/src/tests/testdata/lt/iso-8859-4.txt b/src/tests/testdata/lt/iso-8859-4.txt new file mode 100644 index 0000000..d5ee32a --- /dev/null +++ b/src/tests/testdata/lt/iso-8859-4.txt @@ -0,0 +1,3 @@ +Vincentas van Gogas (ol. Vincent van Gogh, 1853 m. kovo 30 d. Grot Zunderte, +Nyderlandai - 1890 m. liepos 29 d. Overe prie Uazos, Pranczija) - oland +tapytojas ir grafikas, postimpresionistas. diff --git a/src/tests/testdata/lt/utf-8.txt b/src/tests/testdata/lt/utf-8.txt new file mode 100644 index 0000000..de425f9 --- /dev/null +++ b/src/tests/testdata/lt/utf-8.txt @@ -0,0 +1,3 @@ +Vincentas van Gogas (ol. Vincent van Gogh, 1853 m. kovo 30 d. Grot Zunderte, +Nyderlandai – 1890 m. liepos 29 d. Overe prie Uazos, Prancūzija) – olandų +tapytojas ir grafikas, postimpresionistas. diff --git a/src/tests/testdata/lv/iso-8859-10.txt b/src/tests/testdata/lv/iso-8859-10.txt new file mode 100644 index 0000000..6afb063 --- /dev/null +++ b/src/tests/testdata/lv/iso-8859-10.txt @@ -0,0 +1,6 @@ +Vinsents Villems van Gogs (nderlandieu: Vincent Willem van Gogh, dzimis 1853. +gada 30. mart, miris 1890. gada 29. jlij) bija nderlandieu gleznotjs, +postimpresionisma prstvis. Kopum van Gogs radja vairk nek 2000 darbu, to +skait 900 gleznu un 1100 zmjumu un skiu. Savus slavenkos darbus vi radja +pdjo divu dzves gadu laik. Tiek uzskatts, ka van Gogs btiski ir ietekmjis +20. gadsimta mkslu, tostarp ekspresionismu un fovismu. diff --git a/src/tests/testdata/lv/iso-8859-13.txt b/src/tests/testdata/lv/iso-8859-13.txt new file mode 100644 index 0000000..bd4691d --- /dev/null +++ b/src/tests/testdata/lv/iso-8859-13.txt @@ -0,0 +1,6 @@ +Vinsents Villems van Gogs (nderlandieu: Vincent Willem van Gogh, dzimis 1853. +gada 30. mart, miris 1890. gada 29. jlij) bija nderlandieu gleznotjs, +postimpresionisma prstvis. Kopum van Gogs radja vairk nek 2000 darbu, to +skait 900 gleznu un 1100 zmjumu un skiu. Savus slavenkos darbus vi radja +pdjo divu dzves gadu laik. Tiek uzskatts, ka van Gogs btiski ir ietekmjis +20. gadsimta mkslu, tostarp ekspresionismu un fovismu. diff --git a/src/tests/testdata/lv/iso-8859-4.txt b/src/tests/testdata/lv/iso-8859-4.txt new file mode 100644 index 0000000..7fd134d --- /dev/null +++ b/src/tests/testdata/lv/iso-8859-4.txt @@ -0,0 +1,6 @@ +Vinsents Villems van Gogs (nderlandieu: Vincent Willem van Gogh, dzimis 1853. +gada 30. mart, miris 1890. gada 29. jlij) bija nderlandieu gleznotjs, +postimpresionisma prstvis. Kopum van Gogs radja vairk nek 2000 darbu, to +skait 900 gleznu un 1100 zmjumu un skiu. Savus slavenkos darbus vi radja +pdjo divu dzves gadu laik. Tiek uzskatts, ka van Gogs btiski ir ietekmjis +20. gadsimta mkslu, tostarp ekspresionismu un fovismu. diff --git a/src/tests/testdata/lv/utf-8.txt b/src/tests/testdata/lv/utf-8.txt new file mode 100644 index 0000000..4a4d3c9 --- /dev/null +++ b/src/tests/testdata/lv/utf-8.txt @@ -0,0 +1,6 @@ +Vinsents Villems van Gogs (nīderlandiešu: Vincent Willem van Gogh, dzimis 1853. +gada 30. martā, miris 1890. gada 29. jūlijā) bija nīderlandiešu gleznotājs, +postimpresionisma pārstāvis. Kopumā van Gogs radīja vairāk nekā 2000 darbu, to +skaitā 900 gleznu un 1100 zīmējumu un skiču. Savus slavenākos darbus viņš radīja +pēdējo divu dzīves gadu laikā. Tiek uzskatīts, ka van Gogs būtiski ir ietekmējis +20. gadsimta mākslu, tostarp ekspresionismu un fovismu. diff --git a/src/tests/testdata/mt/iso-8859-3.txt b/src/tests/testdata/mt/iso-8859-3.txt new file mode 100644 index 0000000..255269b --- /dev/null +++ b/src/tests/testdata/mt/iso-8859-3.txt @@ -0,0 +1,4 @@ +Franza (Frani:France), uffijalment ir-Repubblika Frania (Frani: +Rpublique franaise), hi pajji fl-Ewropa tal-Punent. Il-belt belt kapitali +tagha hi Parii. Hi membru tal-Unjoni Ewropea. Franza hi maqsuma f'22 rgions +li huma suddivii f' dpartements. diff --git a/src/tests/testdata/mt/utf-8.txt b/src/tests/testdata/mt/utf-8.txt new file mode 100644 index 0000000..079f387 --- /dev/null +++ b/src/tests/testdata/mt/utf-8.txt @@ -0,0 +1,4 @@ +Franza (Franċiż:France), uffiċjalment ir-Repubblika Franċiża (Franċiż: +République française), hi pajjiż fl-Ewropa tal-Punent. Il-belt belt kapitali +tagħha hi Pariġi. Hi membru tal-Unjoni Ewropea. Franza hi maqsuma f'22 régions +li huma suddiviżi f' départements. diff --git a/src/tests/testdata/nl/UTF-8/wikitop_nl_UTF-8.txt b/src/tests/testdata/nl/UTF-8/wikitop_nl_UTF-8.txt deleted file mode 100644 index ac6ba21..0000000 --- a/src/tests/testdata/nl/UTF-8/wikitop_nl_UTF-8.txt +++ /dev/null @@ -1,440 +0,0 @@ -Hoofdpagina - - Uit Wikipedia, de vrije encyclopedie - Ga naar: navigatie, zoeken - Pix.gif - Pix.gif - -Welkom op Wikipedia - - De vrije encyclopedie - - Overzicht van portalen - Pix.gif - De Nederlandstalige Wikipedia telt op dit moment 773.439 artikelen. - Help ook mee door artikelen te starten, te verbeteren of uit te - breiden. - - - - Icoon - - Uitgelicht - - De stripreeks Jommeke is een Vlaamse stripreeks voor de jeugd, getekend - en geschreven door de op 20 oktober 2009 overleden Jef Nys. De strips - vertellen de avonturen van het hoofdpersonage Jommeke, een avontuurlijk - elfjarig jongetje, en een reeks andere personages, zoals zijn vriend - Filiberke, de sprekende papegaai Flip, Professor Gobelijn en de - tweelingzussen De Miekes. - De strip verscheen voor het eerst eind 1955 in het weekblad Kerkelijk - Leven (nu Kerk en Leven), vervolgens in het dagblad Het Volk, en - momenteel verschijnt hij al jarenlang in meerdere kranten. Binnen de - reeks kwamen tot nu toe een 250-tal strips uit. (Lees verder) - - - - Icoon - - Actueel - - Moammar al-Qadhafi - * 20 oktober – De stad Sirte, de laatste Libische stad die in handen - was van Moammar al-Qadhafi, valt in handen van de Nationale - Overgangsraad. Qadhafi wordt gevangengenomen, maar overlijdt aan - verwondingen opgelopen tijdens zijn gevangenneming. - * 19 oktober – De Thaise minister-president Yingluck Shinawatra - verklaart dat haar regering geen controle meer heeft over de - situatie rond de zware overstromingen in 53 provincies, die de - afgelopen weken honderden doden hebben geëist en miljoenen mensen - dakloos hebben gemaakt. - * 18 oktober – Bij gevechten tussen het Filipijnse leger en de - Islamitische afscheidingsbeweging MILF in Al-Barka, in de - zuidelijke Filipijnse provincie Basilan worden minstens 19 soldaten - en 6 rebellen gedood. - * 17 oktober – De Israëlische soldaat Gilad Shalit wordt na vijf jaar - gijzeling vrijgelaten in ruil voor 1027 Palestijnse gevangenen. - * 15 oktober – Het Nederlands honkbalteam behaalt in Panama-Stad voor - de eerste keer in de geschiedenis de wereldtitel door Cuba met 2-1 - te verslaan. - - - - - Icoon - - Wist je dat... - - Het oorspronkelijke Droste-effect - * …het Droste-effect, een internationaal bekende aanduiding voor het - visuele effect waarbij een afbeelding een verkleinde versie van - zichzelf bevat, genoemd is naar een Nederlands cacao-merk? - * …in historisch Tibet belastingafdrachten werden berekend met behulp - van een abacus met losse stenen? - * …Hitler grootse plannen had om Berlijn te herbouwen en te hernoemen - tot Welthauptstadt Germania? - * ...De Indische Waterlelies in de Efteling gebaseerd is op een - sprookje geschreven door Koningin Fabiola van België? - - - - - Icoon - - Recent overleden - - Manfred Gerlach - * Moammar al-Qadhafi (69), Libisch kolonel, 20 oktober - * Manfred Gerlach (83), Duits politicus, laatste voorzitter van de - Staatsraad van de DDR, 17 oktober - * Miguel Ángel Granados Chapa (70), Mexicaans journalist, 16 oktober - * Dan Wheldon (33), Brits autocoureur, 16 oktober - * Adam Hunter (48), Schots golfer, 14 oktober - - - - - Icoon - - 21 oktober in de geschiedenis - - Alfred Nobel - * 1805 - In de slag bij Trafalgar sneuvelt de Britse admiraal Horatio - Nelson. - * 1833 - Geboren: Alfred Nobel, Zweeds chemicus, naamgever van de - Nobelprijs. - * 1920 - Geboren: Coen Ooft, Surinaams politicus. - * 1940 - Geboren: Manfred Mann, Brits muzikant, naamgever van de - gelijknamige R&B band. - * 1945 - Invoering van het stemrecht voor vrouwen in Frankrijk. - * 1948 - Overleden: Koene Dirk Parmentier, Nederlands piloot en - luchtvaartpionier. - * 1959 - Het Guggenheim Museum in New York opent zijn deuren voor het - publiek. - * 1969 - Overleden: Jack Kerouac, Amerikaans schrijver. - * 2006 - Overleden: Paul Biegel, Nederlands kinderboekenschrijver. - - - - - Icoon - - Etalage - - Er staan op dit moment 249 artikelen in de etalage. - __________________________________________________________________ - - Nieuw in de etalage: - * Slot Loevestein - * Maurits van Oranje - - * Spinnen (dieren) - * Turfanosuchus - - Een greep uit de overige etalageartikelen: - * Montréal (Canada) - * Ribbensalamander - * Pape Jan - * Kredietcrisis - * The Who - - * Timaeus (Plato) - * Carl Lewis - * Rioolwaterzuiveringsinstallatie - __________________________________________________________________ - - Etalageaanmeldingen - Richtlijnen voor etalageartikelen - De zes - stappen naar een etalageartikel - - - - Icoon - - Wikipedia in andere talen - - Meer dan 1.000.000 artikelen: English (Engels) – Deutsch (Duits) – - Français (Frans) - - Meer dan 250.000 artikelen: Italiano (Italiaans) – Polski (Pools) – - Español (Spaans) – Русский (Russisch) – 日本語 (Japans) – - Português (Portugees) – Svenska (Zweeds) – 中文 (Chinees) – - Català (Catalaans) – Українська (Oekraïens) – ‪Norsk (bokmål)‬ (Noors) - – Suomi (Fins) - - Meer dan 125.000 artikelen: Tiếng Việt (Vietnamees) – - Česky (Tsjechisch) – Magyar (Hongaars) – 한국어 (Koreaans) – - Türkçe (Turks) – Bahasa Indonesia (Indonesisch) – Română (Roemeens) – - فارسی (Perzisch) - العربية (Arabisch) – Dansk (Deens) – - Esperanto (Esperanto) – Српски / Srpski (Servisch) – Lietuvių (Litouws) - – Slovenčina (Slowaaks) – עברית (Hebreeuws) - - Selectie van andere talen: Afrikaans – Frysk (Fries) – - Lëtzebuergesch (Luxemburgs) – Limburgs – Nedersaksisch – - Papiamentu (Papiaments) – Sranantongo (Surinaams) – Walon (Waals) – - West-Vlams (West-Vlaams) – Zeêuws (Zeeuws) - Alle talen - Portaal van de week - - Portaal van de week - - Film - - Gebruikersportaal, Standaardinformatie, Uitleg - - Over Wikipedia · Hulp & Contact · Helpdocumentatie - Snelcursus: leer Wikipedia bewerken - Hoe kan ik meedoen? · Gebruikersportaal - Navigatie · Artikelindex · Categorieën · Portalen - Overzichten en lijsten · Biografieën - Etalage · Nieuwe artikelen · Gesproken Wikipedia - Conventies van Wikipedia · Neutraal standpunt - Mensenmaatschappij.png - - Mens & Maatschappij, Religie, Politiek - Antropologie · Dagelijks leven · Economie - Filosofie · Occultisme · Openbaar vervoer - Christendom · Islam · Jodendom - Tibetaans boeddhisme · Onderwijs - Geschiedenis, Landen & Volken, Taal - - Geschiedenis, Landen & Volken, Taal - Prehistorie · Oudheid · Middeleeuwen - Tweede Wereldoorlog · Steden - Afrika · Amerika · Azië · Europa · Oceanië - Spanje · Japan · Australië · Italië - Esperanto · Spaans · Latijn - Kunst & Cultuur, Sport - - Kunst & Cultuur, Sport - Film · Literatuur · Muziek · Media - Autosport · Atletiek · Wielersport - Disney · Shakespeare · Strip - Wetenschap & Technologie - - Wetenschap & Technologie - Aardwetenschappen · Astronomie · Biologie - Geneeskunde · Civiele techniek en bouwkunde - Geografie · Informatica · Natuurkunde - Scheikunde · Verkeer en vervoer · Wiskunde - Ruimtevaart · Wetenschapsgeschiedenis · Maritiem - Afbeelding van de dag - - Schimpanse, Pan troglodytes 3.JPG Een chimpansee met jong in Walter - Zoo, een dierentuin in Gossau (Sankt Gallen). - - Zusterprojecten - - Wikipedia is onderdeel van de Wikimedia Foundation, een - non-profitorganisatie, en heeft diverse zusterprojecten die ook van - wikisoftware gebruikmaken: - - Wiktprintable without text.svg - - WikiWoordenboek - Vrij woordenboek met betekenissen, vertalingen, etymologie en uitspraak - - Wikibooks-logo.svg - - Wikibooks - Handleidingen en vrije boeken - - WikiquoteL.jpg - - Wikiquote - Een vrije collectie van citaten en spreekwoorden - - WikinotizieLnl.png - - Wikinews - Vrije nieuwsbron (Engelstalig) - - WikisourceL.jpg - - Wikisource - Documenten uit het publieke domein - - WikispeciesL.jpg - - Wikispecies - Catalogus van alle soorten levende wezens - - CommonsL.jpg - - Commons - Centrale opslagplaats voor mediabestanden van de projecten - - Wikimedia Community Logo.svg - - Meta-Wiki - Internationale website over de projecten van Wikimedia - - WikiversityL.jpg - - Wikiversity - Verzameling van vrije onderwijsprojecten - - IncubatorL.png - - Incubator - Opstarten van nieuwe projecten - - Perscontact · Publiekscontact · Statistieken - Ontvangen van - "http://nl.wikipedia.org/w/index.php?title=Hoofdpagina&oldid=25351209" - Categorie: - * Alles - -Persoonlijke instellingen - - * Aanmelden / registreren - -Naamruimten - - * Artikel - * Overleg - -Varianten - -Weergaven - - * Lezen - * Brontekst bekijken - * Geschiedenis - -Handelingen - -Zoeken - - ____________________ (Submit) Zoeken - -Navigatie - - * Hoofdpagina - * Vind een artikel - * Vandaag - * Etalage - * Categorieën - * Recente wijzigingen - * Nieuwe artikelen - * Willekeurige pagina - -Informatie - - * Gebruikersportaal - * Snelcursus - * Hulp en contact - * Donaties - -Hulpmiddelen - - * Links naar deze pagina - * Verwante wijzigingen - * Bestand uploaden - * Speciale pagina's - * Permanente verwijzing - * Deze pagina citeren - -Afdrukken/exporteren - - * Boek maken - * Downloaden als PDF - * Printervriendelijke versie - -In andere talen - - * Afrikaans - * Alemannisch - * Ænglisc - * العربية - * ܐܪܡܝܐ - * Aymar aru - * Беларуская - * Български - * Català - * Česky - * Cymraeg - * Dansk - * Deutsch - * Ελληνικά - * English - * Esperanto - * Español - * Eesti - * فارسی - * Suomi - * Võro - * Na Vosa Vakaviti - * Français - * Frysk - * Gaeilge - * Galego - * Hawai`i - * עברית - * Hrvatski - * Magyar - * Bahasa Indonesia - * Ido - * Íslenska - * Italiano - * 日本語 - * Kalaallisut - * 한국어 - * Kurdî - * Latina - * Lëtzebuergesch - * Limburgs - * Lietuvių - * Latviešu - * Māori - * Dorerin Naoero - * Nedersaksisch - * ‪Norsk (nynorsk)‬ - * ‪Norsk (bokmål)‬ - * Papiamentu - * Deitsch - * Norfuk / Pitkern - * Polski - * پښتو - * Português - * Română - * Русский - * Simple English - * Slovenčina - * Slovenščina - * Gagana Samoa - * Shqip - * Српски / Srpski - * Sranantongo - * Svenska - * Kiswahili - * తెలుగు - * ไทย - * lea faka-Tonga - * Tok Pisin - * Türkçe - * Татарча/Tatarça - * Reo Mā`ohi - * Українська - * Tiếng Việt - * West-Vlams - * Walon - * Winaray - * 吴语 - * Zeêuws - * 中文 - * 文言 - - * Deze pagina is het laatst bewerkt op 25 apr 2011 om 13:03. - * De tekst is beschikbaar onder de licentie Creative Commons - Naamsvermelding/Gelijk delen, er kunnen aanvullende voorwaarden van - toepassing zijn. Zie de Gebruiksvoorwaarden voor meer informatie. - Wikipedia® is een geregistreerd handelsmerk van de Wikimedia - Foundation, Inc., een organisatie zonder winstoogmerk. - - * Privacybeleid - * Over Wikipedia - * Voorbehoud - * Mobiele weergave - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/nl/WINDOWS-1252/wikitop_nl_WINDOWS-1252.txt b/src/tests/testdata/nl/WINDOWS-1252/wikitop_nl_WINDOWS-1252.txt deleted file mode 100644 index bfe960a..0000000 --- a/src/tests/testdata/nl/WINDOWS-1252/wikitop_nl_WINDOWS-1252.txt +++ /dev/null @@ -1,437 +0,0 @@ -Hoofdpagina - - Uit Wikipedia, de vrije encyclopedie - Ga naar: navigatie, zoeken - -Welkom op Wikipedia - - De vrije encyclopedie - - Overzicht van portalen - De Nederlandstalige Wikipedia telt op dit moment 773.439 artikelen. - Help ook mee door artikelen te starten, te verbeteren of uit te - breiden. - - - - Icoon - - Uitgelicht - - De stripreeks Jommeke is een Vlaamse stripreeks voor de jeugd, getekend - en geschreven door de op 20 oktober 2009 overleden Jef Nys. De strips - vertellen de avonturen van het hoofdpersonage Jommeke, een avontuurlijk - elfjarig jongetje, en een reeks andere personages, zoals zijn vriend - Filiberke, de sprekende papegaai Flip, Professor Gobelijn en de - tweelingzussen De Miekes. - De strip verscheen voor het eerst eind 1955 in het weekblad Kerkelijk - Leven (nu Kerk en Leven), vervolgens in het dagblad Het Volk, en - momenteel verschijnt hij al jarenlang in meerdere kranten. Binnen de - reeks kwamen tot nu toe een 250-tal strips uit. (Lees verder) - - - - Icoon - - Actueel - - Moammar al-Qadhafi - * 20 oktober De stad Sirte, de laatste Libische stad die in handen - was van Moammar al-Qadhafi, valt in handen van de Nationale - Overgangsraad. Qadhafi wordt gevangengenomen, maar overlijdt aan - verwondingen opgelopen tijdens zijn gevangenneming. - * 19 oktober De Thaise minister-president Yingluck Shinawatra - verklaart dat haar regering geen controle meer heeft over de - situatie rond de zware overstromingen in 53 provincies, die de - afgelopen weken honderden doden hebben geist en miljoenen mensen - dakloos hebben gemaakt. - * 18 oktober Bij gevechten tussen het Filipijnse leger en de - Islamitische afscheidingsbeweging MILF in Al-Barka, in de - zuidelijke Filipijnse provincie Basilan worden minstens 19 soldaten - en 6 rebellen gedood. - * 17 oktober De Isralische soldaat Gilad Shalit wordt na vijf jaar - gijzeling vrijgelaten in ruil voor 1027 Palestijnse gevangenen. - * 15 oktober Het Nederlands honkbalteam behaalt in Panama-Stad voor - de eerste keer in de geschiedenis de wereldtitel door Cuba met 2-1 - te verslaan. - - - - - Icoon - - Wist je dat... - - Het oorspronkelijke Droste-effect - * het Droste-effect, een internationaal bekende aanduiding voor het - visuele effect waarbij een afbeelding een verkleinde versie van - zichzelf bevat, genoemd is naar een Nederlands cacao-merk? - * in historisch Tibet belastingafdrachten werden berekend met behulp - van een abacus met losse stenen? - * Hitler grootse plannen had om Berlijn te herbouwen en te hernoemen - tot Welthauptstadt Germania? - * ...De Indische Waterlelies in de Efteling gebaseerd is op een - sprookje geschreven door Koningin Fabiola van Belgi? - - - - - Icoon - - Recent overleden - - Manfred Gerlach - * Moammar al-Qadhafi (69), Libisch kolonel, 20 oktober - * Manfred Gerlach (83), Duits politicus, laatste voorzitter van de - Staatsraad van de DDR, 17 oktober - * Miguel ngel Granados Chapa (70), Mexicaans journalist, 16 oktober - * Dan Wheldon (33), Brits autocoureur, 16 oktober - * Adam Hunter (48), Schots golfer, 14 oktober - - - - - Icoon - - 21 oktober in de geschiedenis - - Alfred Nobel - * 1805 - In de slag bij Trafalgar sneuvelt de Britse admiraal Horatio - Nelson. - * 1833 - Geboren: Alfred Nobel, Zweeds chemicus, naamgever van de - Nobelprijs. - * 1920 - Geboren: Coen Ooft, Surinaams politicus. - * 1940 - Geboren: Manfred Mann, Brits muzikant, naamgever van de - gelijknamige R&B band. - * 1945 - Invoering van het stemrecht voor vrouwen in Frankrijk. - * 1948 - Overleden: Koene Dirk Parmentier, Nederlands piloot en - luchtvaartpionier. - * 1959 - Het Guggenheim Museum in New York opent zijn deuren voor het - publiek. - * 1969 - Overleden: Jack Kerouac, Amerikaans schrijver. - * 2006 - Overleden: Paul Biegel, Nederlands kinderboekenschrijver. - - - - - Icoon - - Etalage - - Er staan op dit moment 249 artikelen in de etalage. - __________________________________________________________________ - - Nieuw in de etalage: - * Slot Loevestein - * Maurits van Oranje - - * Spinnen (dieren) - * Turfanosuchus - - Een greep uit de overige etalageartikelen: - * Montral (Canada) - * Ribbensalamander - * Pape Jan - * Kredietcrisis - * The Who - - * Timaeus (Plato) - * Carl Lewis - * Rioolwaterzuiveringsinstallatie - __________________________________________________________________ - - Etalageaanmeldingen - Richtlijnen voor etalageartikelen - De zes - stappen naar een etalageartikel - - - - Icoon - - Wikipedia in andere talen - - Meer dan 1.000.000 artikelen: English (Engels) Deutsch (Duits) - Franais (Frans) - - Meer dan 250.000 artikelen: Italiano (Italiaans) Polski (Pools) - Espaol (Spaans) (Russisch) (Japans) - Portugus (Portugees) Svenska (Zweeds) (Chinees) - Catal (Catalaans) (Oekraens) Norsk (bokml) (Noors) - Suomi (Fins) - - Meer dan 125.000 artikelen: Ting Vit (Vietnamees) - esky (Tsjechisch) Magyar (Hongaars) (Koreaans) - Trke (Turks) Bahasa Indonesia (Indonesisch) Romn (Roemeens) - (Perzisch) - (Arabisch) Dansk (Deens) - Esperanto (Esperanto) / Srpski (Servisch) Lietuvi (Litouws) - Slovenina (Slowaaks) (Hebreeuws) - - Selectie van andere talen: Afrikaans Frysk (Fries) - Ltzebuergesch (Luxemburgs) Limburgs Nedersaksisch - Papiamentu (Papiaments) Sranantongo (Surinaams) Walon (Waals) - West-Vlams (West-Vlaams) Zeuws (Zeeuws) - Alle talen - Portaal van de week - - Portaal van de week - - Film - - Gebruikersportaal, Standaardinformatie, Uitleg - - Over Wikipedia Hulp & Contact Helpdocumentatie - Snelcursus: leer Wikipedia bewerken - Hoe kan ik meedoen? Gebruikersportaal - Navigatie Artikelindex Categorien Portalen - Overzichten en lijsten Biografien - Etalage Nieuwe artikelen Gesproken Wikipedia - Conventies van Wikipedia Neutraal standpunt - Mensenmaatschappij.png - - Mens & Maatschappij, Religie, Politiek - Antropologie Dagelijks leven Economie - Filosofie Occultisme Openbaar vervoer - Christendom Islam Jodendom - Tibetaans boeddhisme Onderwijs - Geschiedenis, Landen & Volken, Taal - - Geschiedenis, Landen & Volken, Taal - Prehistorie Oudheid Middeleeuwen - Tweede Wereldoorlog Steden - Afrika Amerika Azi Europa Oceani - Spanje Japan Australi Itali - Esperanto Spaans Latijn - Kunst & Cultuur, Sport - - Kunst & Cultuur, Sport - Film Literatuur Muziek Media - Autosport Atletiek Wielersport - Disney Shakespeare Strip - Wetenschap & Technologie - - Wetenschap & Technologie - Aardwetenschappen Astronomie Biologie - Geneeskunde Civiele techniek en bouwkunde - Geografie Informatica Natuurkunde - Scheikunde Verkeer en vervoer Wiskunde - Ruimtevaart Wetenschapsgeschiedenis Maritiem - Afbeelding van de dag - - Schimpanse, Pan troglodytes 3.JPG Een chimpansee met jong in Walter - Zoo, een dierentuin in Gossau (Sankt Gallen). - - Zusterprojecten - - Wikipedia is onderdeel van de Wikimedia Foundation, een - non-profitorganisatie, en heeft diverse zusterprojecten die ook van - wikisoftware gebruikmaken: - - Wiktprintable without text.svg - - WikiWoordenboek - Vrij woordenboek met betekenissen, vertalingen, etymologie en uitspraak - - Wikibooks-logo.svg - - Wikibooks - Handleidingen en vrije boeken - - WikiquoteL.jpg - - Wikiquote - Een vrije collectie van citaten en spreekwoorden - - WikinotizieLnl.png - - Wikinews - Vrije nieuwsbron (Engelstalig) - - WikisourceL.jpg - - Wikisource - Documenten uit het publieke domein - - WikispeciesL.jpg - - Wikispecies - Catalogus van alle soorten levende wezens - - CommonsL.jpg - - Commons - Centrale opslagplaats voor mediabestanden van de projecten - - Wikimedia Community Logo.svg - - Meta-Wiki - Internationale website over de projecten van Wikimedia - - WikiversityL.jpg - - Wikiversity - Verzameling van vrije onderwijsprojecten - - IncubatorL.png - - Incubator - Opstarten van nieuwe projecten - - Perscontact Publiekscontact Statistieken - Ontvangen van - "http://nl.wikipedia.org/w/index.php?title=Hoofdpagina&oldid=25351209" - Categorie: - * Alles - -Persoonlijke instellingen - - * Aanmelden / registreren - -Naamruimten - - * Artikel - * Overleg - -Varianten - -Weergaven - - * Lezen - * Brontekst bekijken - * Geschiedenis - -Handelingen - -Zoeken - - ____________________ (Submit) Zoeken - -Navigatie - - * Hoofdpagina - * Vind een artikel - * Vandaag - * Etalage - * Categorien - * Recente wijzigingen - * Nieuwe artikelen - * Willekeurige pagina - -Informatie - - * Gebruikersportaal - * Snelcursus - * Hulp en contact - * Donaties - -Hulpmiddelen - - * Links naar deze pagina - * Verwante wijzigingen - * Bestand uploaden - * Speciale pagina's - * Permanente verwijzing - * Deze pagina citeren - -Afdrukken/exporteren - - * Boek maken - * Downloaden als PDF - * Printervriendelijke versie - -In andere talen - - * Afrikaans - * Alemannisch - * nglisc - * - * - * Aymar aru - * - * - * Catal - * esky - * Cymraeg - * Dansk - * Deutsch - * - * English - * Esperanto - * Espaol - * Eesti - * - * Suomi - * Vro - * Na Vosa Vakaviti - * Franais - * Frysk - * Gaeilge - * Galego - * Hawai`i - * - * Hrvatski - * Magyar - * Bahasa Indonesia - * Ido - * slenska - * Italiano - * - * Kalaallisut - * - * Kurd - * Latina - * Ltzebuergesch - * Limburgs - * Lietuvi - * Latvieu - * Mori - * Dorerin Naoero - * Nedersaksisch - * Norsk (nynorsk) - * Norsk (bokml) - * Papiamentu - * Deitsch - * Norfuk / Pitkern - * Polski - * - * Portugus - * Romn - * - * Simple English - * Slovenina - * Slovenina - * Gagana Samoa - * Shqip - * / Srpski - * Sranantongo - * Svenska - * Kiswahili - * - * - * lea faka-Tonga - * Tok Pisin - * Trke - * /Tatara - * Reo M`ohi - * - * Ting Vit - * West-Vlams - * Walon - * Winaray - * - * Zeuws - * - * - - * Deze pagina is het laatst bewerkt op 25 apr 2011 om 13:03. - * De tekst is beschikbaar onder de licentie Creative Commons - Naamsvermelding/Gelijk delen, er kunnen aanvullende voorwaarden van - toepassing zijn. Zie de Gebruiksvoorwaarden voor meer informatie. - Wikipedia is een geregistreerd handelsmerk van de Wikimedia - Foundation, Inc., een organisatie zonder winstoogmerk. - - * Privacybeleid - * Over Wikipedia - * Voorbehoud - * Mobiele weergave - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/no/UTF-8/wikitop_no_UTF-8.txt b/src/tests/testdata/no/UTF-8/wikitop_no_UTF-8.txt deleted file mode 100644 index 045e4fb..0000000 --- a/src/tests/testdata/no/UTF-8/wikitop_no_UTF-8.txt +++ /dev/null @@ -1,251 +0,0 @@ -Portal:Forside - - Fra Wikipedia, den frie encyklopedi - Gå til: navigasjon, søk - Velkommen til Wikipedia, - - den frie encyklopedi som du kan forbedre. - 315 168 artikler på bokmål og riksmål - - Bla gjennom kategoriene · Alfabetisk liste · Portaler - Kategorier: Kultur · Geografi · Historie · Matematikk · Vitenskap · - Samfunn - Portaler: Kunst · Film · Historie · Samfunn · Geografi · Vitenskap · - Språk - -Ukens artikkel (uke 42) - - Tittelbladet i Christian IIIs Bibel, fra 1550 - - Historien om Bibelen i Norge begynner på slutten av 1200-tallet, da - deler av Det gamle testamente ble oversatt til gammelnorsk med navnet - Stjórn. Christian IIs nytestamente (1524) var den neste bibelteksten - som kom ut i Danmark-Norge. Christian II var i 1524 i Wittenberg, der - han ble inspirert av reformasjonen. - - Enkelte bibeloversettelser har fått større betydning enn andre. For - noen, som for den nordsamiske oversettelsen fra 1895, kommer dette av - at de har vært enerådende over lang tid. For andre, som Bibelselskapets - oversettelse fra 1930, kommer det av at det er svært gode - oversettelser. I 1891 kom den første hele oversettelsen av GT til - norsk, gjort direkte fra grunnspråkene. De første bibeltekstene på - landsmål kom i 1859, da Ivar Aasen oversatte historien om den bortkomne - sønnen. Fyrebilsbibelen, som var den første hele bibeloversettelsen på - nynorsk, kom i 1921. I oktober 2011 utkommer Bibel 2011, en ny - oversettelse fra Det Norske Bibelselskap. Les mer… - - Om ukens artikkel · Arkiv over årets artikler · 209 utmerkede artikler - -Dagens skandinaviske artikkel (dansk) - - Winston Churchill, 1942 - - Winston Churchill (1874 - 1965) var en var en britisk politiker og - statsmand, som især er kendt for sit lederskab af Storbritannien under - 2. verdenskrig. Han var premierminister i Storbritannien 1940–45 og - 1951–55. Han betragtes i vide kredse som en af de største ledere i - krigstid. - - Ved siden af sit politiske arbejde var han forfatter til 37 bøger, og - han nåede at male over 500 malerier. Som forfatter skrev han navnlig - populærhistoriske værker samt oversigtsværker over samtiden. Han er den - eneste britiske premierminister, som har modtaget Nobelprisen i - litteratur, og han var den første, som blev udnævnt til æresborger i - USA. - - Churchill blev født ind i en aristokratisk familie. Hans far Randolph - Churchill, som var efterkommer af hertugen af Marlborough, var en - karismatisk politiker, som bl.a. var britisk finansminister. Hans mor, - Jenny Jerome, stammede fra en velhavende amerikansk familie. Læs mere - - Arkiv · Dette er ukens artikkel på dansk Wikipedia - -Søsterprosjekter - - Wikipedia utgis av den ideelle organisasjonen Wikimedia Foundation, som - driver flere flerspråklige og frie prosjekter hvor alle kan bidra. - - - Wikimedia Community Logo.svg Meta-Wiki - Koordinasjonsside for Wikimedia-prosjekter Commons-logo.svg Commons - Lagringssted for delt multimedieinnhold - Wiktionary-logo-en.png Wiktionary - Fri ordbok Wikibooks-logo.svg Wikibøker - Bøker og manualer - Wikiquote-logo.svg Wikiquote - Sitatsamling Wikisource-logo.svg Wikikilden - Originale tekster - Wikispecies-logo.svg Wikispecies - Oppslagsverk over dyre- og plantearter Wikinews-logo.png Wikinytt - Nyhetstjeneste med fritt innhold - -Om Wikipedia - - Wikipedia er en encyklopedi på mer enn 270 språk, som skrives av - frivillige bidragsytere fra hele verden. Innholdet kan kopieres og - brukes fritt. Vi startet i 2001 og arbeider nå på mer enn 19 millioner - artikler, hvorav 315 168 i den norske utgaven på bokmål og riksmål. På - den nynorske versjonen arbeides det på over 72 000 artikler. Les mer om - hvordan du kan bidra! - -Aktuelt - - Muammar al-Gaddafi at the AU summit.jpg - * Libyas tidligere leder Muammar al-Gaddafi (bildet) blir drept i - Surt. - - * Julian Barnes får Bookerprisen for romanen The Sense of an Ending. - - * Slåttehumle fotografert i Norge for første gang. - - * Nordisk Råds filmpris tildeles den svenske filmen Skyggesiden. - __________________________________________________________________ - - Bakgrunn: Terrorangrepene i Norge • Opprør i den arabiske verden - Kultur: Bergen internasjonale filmfestival - Sport: Panamerikanske leker - -I dag - - Den 21. oktober: - Florence Nightingale - * 1805 – Napoleonskrigene: Napoleon måtte oppgi planene om invasjon - av England etter tapet i Slaget ved Trafalgar. - * 1833 – Alfred Nobel ble født. - * 1854 – Florence Nightingale (bildet) og en stab på 38 sykepleiere - ble sendt til Krimkrigen. - * 1944 – Andre verdenskrig: Slaget om Aachen tok slutt da de - gjenværende tyske styrkene overga seg til amerikanerne. - * 1962 – Hurtigruteskipet MS «Sanct Svithun» forliste ved Nordøyan - fyr på Folda. - * 1983 – CGPM redefinerte en meter som den avstanden lyset - tilbakelegger i et vakuum i løpet av 1/299 792 458 av et sekund. - * 1984 – Den franske filmskaperen François Truffaut døde. - -Nylig avdøde - - Muammar al-Gaddafi - * 20. oktober – Muammar al-Gaddafi (bildet), libysk diktator (69) - * 17. oktober – Benito Nava, italienskfødt restauratør og musiker - (75) - * 10. oktober – Ulf Löfgren, svensk barnebokkunstner (79) - - Flere nylig avdøde - -Visste du at ... - - Ringmerke - * … at mer enn 7 millioner fugl er ringmerket (bildet) i Norge? - * … John Eliot (1604–90) var den første som oversatte Bibelen til et - indianerspråk? - * … escolar eller «smørfisk» er populær i sushi, men har vært forbudt - i Japan siden 1977? - - Fakta ovenfor er hentet fra noen av de nyeste artiklene våre. - - Hentet fra - «http://no.wikipedia.org/w/index.php?title=Portal:Forside&oldid=8939782 - » - Kategori: - * Wikipedia - -Personlig - - * Logg inn eller opprett en konto - -Navnerom - - * Portal - * Diskusjon - -Varianter - -Visninger - - * Les - * Vis kildetekst - * Vis historikk - -Handlinger - -Søk - - ____________________ (Submit) Søk - -Navigasjon - - * Forside - * Svaksynte - * Anbefalte artikler - * Utmerkede artikler - * Hjelpesider - * Siste endringer - -Prosjekt - - * Prosjektportal - * Tinget - * Tilfeldig side - -Wikipedia - - * Kontakt Wikipedia - * Donasjoner - * Wikimedia Norge - -Andre - -Eksternt - -Lager - -Utskrift - - * Opprett en bok - * Last ned som PDF - * Utskriftsversjon - -Verktøy - - * Lenker hit - * Relaterte endringer - * Spesialsider - * Permanent lenke - -På andre språk - - * ‪Norsk (nynorsk)‬ - * Dansk - * Svenska - * Català - * Deutsch - * English - * Español - * Suomi - * Føroyskt - * Français - * Íslenska - * Italiano - * 日本語 - * Nederlands - * Polski - * Português - * Română - * Русский - * Sámegiella - * Türkçe - * Українська - * 中文 - - * Denne siden ble sist endret 14. jun 2011 kl. 21:30. - * Innholdet er tilgjengelig under Creative Commons-lisensen - Navngivelse-Del på samme vilkår, men ytterligere betingelser kan - gjelde. Se bruksvilkårene for detaljer. - - * Personvern - * Om Wikipedia - * Forbehold - * Mobilvisning - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/no/WINDOWS-1252/wikitop_no_WINDOWS-1252.txt b/src/tests/testdata/no/WINDOWS-1252/wikitop_no_WINDOWS-1252.txt deleted file mode 100644 index 6f21bb3..0000000 --- a/src/tests/testdata/no/WINDOWS-1252/wikitop_no_WINDOWS-1252.txt +++ /dev/null @@ -1,253 +0,0 @@ - #Wikipedia (no) copyright Wikipedia Atom-mating - -Portal:Forside - - Fra Wikipedia, den frie encyklopedi - G til: navigasjon, sk - Velkommen til Wikipedia, - - den frie encyklopedi som du kan forbedre. - 315 168 artikler p bokml og riksml - - Bla gjennom kategoriene Alfabetisk liste Portaler - Kategorier: Kultur Geografi Historie Matematikk Vitenskap - Samfunn - Portaler: Kunst Film Historie Samfunn Geografi Vitenskap - Sprk - -Ukens artikkel (uke 42) - - Tittelbladet i Christian IIIs Bibel, fra 1550 - - Historien om Bibelen i Norge begynner p slutten av 1200-tallet, da - deler av Det gamle testamente ble oversatt til gammelnorsk med navnet - Stjrn. Christian IIs nytestamente (1524) var den neste bibelteksten - som kom ut i Danmark-Norge. Christian II var i 1524 i Wittenberg, der - han ble inspirert av reformasjonen. - - Enkelte bibeloversettelser har ftt strre betydning enn andre. For - noen, som for den nordsamiske oversettelsen fra 1895, kommer dette av - at de har vrt enerdende over lang tid. For andre, som Bibelselskapets - oversettelse fra 1930, kommer det av at det er svrt gode - oversettelser. I 1891 kom den frste hele oversettelsen av GT til - norsk, gjort direkte fra grunnsprkene. De frste bibeltekstene p - landsml kom i 1859, da Ivar Aasen oversatte historien om den bortkomne - snnen. Fyrebilsbibelen, som var den frste hele bibeloversettelsen p - nynorsk, kom i 1921. I oktober 2011 utkommer Bibel 2011, en ny - oversettelse fra Det Norske Bibelselskap. Les mer - - Om ukens artikkel Arkiv over rets artikler 209 utmerkede artikler - -Dagens skandinaviske artikkel (dansk) - - Winston Churchill, 1942 - - Winston Churchill (1874 - 1965) var en var en britisk politiker og - statsmand, som isr er kendt for sit lederskab af Storbritannien under - 2. verdenskrig. Han var premierminister i Storbritannien 194045 og - 195155. Han betragtes i vide kredse som en af de strste ledere i - krigstid. - - Ved siden af sit politiske arbejde var han forfatter til 37 bger, og - han nede at male over 500 malerier. Som forfatter skrev han navnlig - populrhistoriske vrker samt oversigtsvrker over samtiden. Han er den - eneste britiske premierminister, som har modtaget Nobelprisen i - litteratur, og han var den frste, som blev udnvnt til resborger i - USA. - - Churchill blev fdt ind i en aristokratisk familie. Hans far Randolph - Churchill, som var efterkommer af hertugen af Marlborough, var en - karismatisk politiker, som bl.a. var britisk finansminister. Hans mor, - Jenny Jerome, stammede fra en velhavende amerikansk familie. Ls mere - - Arkiv Dette er ukens artikkel p dansk Wikipedia - -Ssterprosjekter - - Wikipedia utgis av den ideelle organisasjonen Wikimedia Foundation, som - driver flere flersprklige og frie prosjekter hvor alle kan bidra. - - - Wikimedia Community Logo.svg Meta-Wiki - Koordinasjonsside for Wikimedia-prosjekter Commons-logo.svg Commons - Lagringssted for delt multimedieinnhold - Wiktionary-logo-en.png Wiktionary - Fri ordbok Wikibooks-logo.svg Wikibker - Bker og manualer - Wikiquote-logo.svg Wikiquote - Sitatsamling Wikisource-logo.svg Wikikilden - Originale tekster - Wikispecies-logo.svg Wikispecies - Oppslagsverk over dyre- og plantearter Wikinews-logo.png Wikinytt - Nyhetstjeneste med fritt innhold - -Om Wikipedia - - Wikipedia er en encyklopedi p mer enn 270 sprk, som skrives av - frivillige bidragsytere fra hele verden. Innholdet kan kopieres og - brukes fritt. Vi startet i 2001 og arbeider n p mer enn 19 millioner - artikler, hvorav 315 168 i den norske utgaven p bokml og riksml. P - den nynorske versjonen arbeides det p over 72 000 artikler. Les mer om - hvordan du kan bidra! - -Aktuelt - - Muammar al-Gaddafi at the AU summit.jpg - * Libyas tidligere leder Muammar al-Gaddafi (bildet) blir drept i - Surt. - - * Julian Barnes fr Bookerprisen for romanen The Sense of an Ending. - - * Slttehumle fotografert i Norge for frste gang. - - * Nordisk Rds filmpris tildeles den svenske filmen Skyggesiden. - __________________________________________________________________ - - Bakgrunn: Terrorangrepene i Norge Opprr i den arabiske verden - Kultur: Bergen internasjonale filmfestival - Sport: Panamerikanske leker - -I dag - - Den 21. oktober: - Florence Nightingale - * 1805 Napoleonskrigene: Napoleon mtte oppgi planene om invasjon - av England etter tapet i Slaget ved Trafalgar. - * 1833 Alfred Nobel ble fdt. - * 1854 Florence Nightingale (bildet) og en stab p 38 sykepleiere - ble sendt til Krimkrigen. - * 1944 Andre verdenskrig: Slaget om Aachen tok slutt da de - gjenvrende tyske styrkene overga seg til amerikanerne. - * 1962 Hurtigruteskipet MS Sanct Svithun forliste ved Nordyan - fyr p Folda. - * 1983 CGPM redefinerte en meter som den avstanden lyset - tilbakelegger i et vakuum i lpet av 1/299 792 458 av et sekund. - * 1984 Den franske filmskaperen Franois Truffaut dde. - -Nylig avdde - - Muammar al-Gaddafi - * 20. oktober Muammar al-Gaddafi (bildet), libysk diktator (69) - * 17. oktober Benito Nava, italienskfdt restauratr og musiker - (75) - * 10. oktober Ulf Lfgren, svensk barnebokkunstner (79) - - Flere nylig avdde - -Visste du at ... - - Ringmerke - * at mer enn 7 millioner fugl er ringmerket (bildet) i Norge? - * John Eliot (160490) var den frste som oversatte Bibelen til et - indianersprk? - * escolar eller smrfisk er populr i sushi, men har vrt forbudt - i Japan siden 1977? - - Fakta ovenfor er hentet fra noen av de nyeste artiklene vre. - - Hentet fra - http://no.wikipedia.org/w/index.php?title=Portal:Forside&oldid=8939782 - - Kategori: - * Wikipedia - -Personlig - - * Logg inn eller opprett en konto - -Navnerom - - * Portal - * Diskusjon - -Varianter - -Visninger - - * Les - * Vis kildetekst - * Vis historikk - -Handlinger - -Sk - - ____________________ (Submit) Sk - -Navigasjon - - * Forside - * Svaksynte - * Anbefalte artikler - * Utmerkede artikler - * Hjelpesider - * Siste endringer - -Prosjekt - - * Prosjektportal - * Tinget - * Tilfeldig side - -Wikipedia - - * Kontakt Wikipedia - * Donasjoner - * Wikimedia Norge - -Andre - -Eksternt - -Lager - -Utskrift - - * Opprett en bok - * Last ned som PDF - * Utskriftsversjon - -Verkty - - * Lenker hit - * Relaterte endringer - * Spesialsider - * Permanent lenke - -P andre sprk - - * Norsk (nynorsk) - * Dansk - * Svenska - * Catal - * Deutsch - * English - * Espaol - * Suomi - * Froyskt - * Franais - * slenska - * Italiano - * - * Nederlands - * Polski - * Portugus - * Romn - * - * Smegiella - * Trke - * - * - - * Denne siden ble sist endret 14. jun 2011 kl. 21:30. - * Innholdet er tilgjengelig under Creative Commons-lisensen - Navngivelse-Del p samme vilkr, men ytterligere betingelser kan - gjelde. Se bruksvilkrene for detaljer. - - * Personvern - * Om Wikipedia - * Forbehold - * Mobilvisning - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/pl/ISO-8859-2/wikitop_pl_ISO-8859-2.txt b/src/tests/testdata/pl/ISO-8859-2/wikitop_pl_ISO-8859-2.txt deleted file mode 100644 index f5626e1..0000000 --- a/src/tests/testdata/pl/ISO-8859-2/wikitop_pl_ISO-8859-2.txt +++ /dev/null @@ -1,272 +0,0 @@ -Strona gwna - - Z Wikipedii, wolnej encyklopedii - Skocz do: nawigacji, szukaj - - Nauki cise i przyrodnicze Nauki spoeczne i humanistyczne - Technika Geografia - Historia Kultura Spoeczestwo Sport Religie Polska - Wicej portali Indeks hase Przegld kategorii - - Witaj w Wikipedii wolnej encyklopedii, ktr kady moe redagowa. - - W polskiej wersji mamy 837 872 hasa. - Gromadzimy i udostpniamy wiedz. - Docz do nas! - -Czy wiesz... - - Z nowych artykuw w Wikipedii: - Slushko Palace.jpg - - na jakim pwyspie stan wileski Paac Suszkw? - - dlaczego groono mierci amerykaskiej socjoloce Idzie - Wells-Barnett? - Antonowka 2.jpg - - jak zakoczya si obrona Antonwki, orodka polskiej samoobrony na - Woyniu? - - jakim wydarzeniom w 1920 roku powicony by kalendarzyk - ekspresjonistyczny? - - w ktrych obozach koncentracyjnych by wiziony Teodor Hilary - Drapiewski? - Zaproponuj artyku do ekspozycji Jak napisa nowe haso? - -Wydarzenia - - Gd w Rogu Afryki Puchar wiata w rugby - __________________________________________________________________ - - Muammar al-Kaddafi, 2 lutego 2009 - * W okolicach Syrty zgin byy przywdca Libii Muammar al-Kaddafi - (na zdjciu). (20 padziernika) - - * W wyniku negocjacji Izraela z wadzami Autonomii Palestyskiej, - Gilad Szalit zosta uwolniony przez Hamas w zamian za 1027 - Palestyczykw. (18 padziernika) - - * W trakcie wycigu Indy Racing League doszo do karambolu, w ktrym - zgin Dan Wheldon. (16 padziernika) - - * W Moskwie odsonito pierwszy w Rosji pomnik Jana Pawa II. Ma - ponad 1,8 m wysokoci i way dwie tony. (14 padziernika) - - * Julia Tymoszenko zostaa uznana za winn naduy i skazana na - siedem lat wizienia. (11 padziernika) - - Wicej: w Wikipedii, w Wikinews - -Rocznice - - 21 padziernika: imieniny obchodz Celina, Jakub, Urszula - Willy Brandt - * 1824 Joseph Aspdin opatentowa cement portlandzki - * 1833 urodzi si Alfred Nobel, szwedzki chemik, wynalazca i - filantrop, fundator Nagrody Nobla - * 1929 urodzia si Ursula K. Le Guin, amerykaska pisarka - fantastyki - * 1956 I sekretarzem PZPR w miejsce Edwarda Ochaba zosta Wadysaw - Gomuka - * 1963 w obawie SB i ZOMO zgin Jzef Franczak ps. Lalek, ostatni - onierz polskiego podziemia - * 1969 zmar Wacaw Sierpiski, polski matematyk - * 1969 Willy Brandt (na zdjciu), przywdca SPD, zosta kanclerzem - Niemieckiej Republiki Federalnej - - 20 padziernika Kalendarium dzie po dniu 22 padziernika - -Artyku na medal - - Bazyli doroszkiewicz.jpg - Bazyli, imi wieckie: Wodzimierz Doroszkiewicz (ur. 15 marca 1914 w - Cisach w Prusach Wschodnich, zm. 11 lutego 1998 w Warszawie) pity - metropolita warszawski i caej Polski, zwierzchnik Polskiego - Autokefalicznego Kocioa Prawosawnego od 1970 do mierci w 1998. W - 1938 przyj wicenia kapaskie. Do 1960 prowadzi prac - duszpastersk w rnych parafiach na Biaostocczynie oraz - Grodzieszczynie. W grudniu 1959 zoy wieczyste luby zakonne, - przyjmujc imi Bazyli. W 1970 obj urzd metropolity warszawskiego i - caej Polski. W okresie penienia przez niego zwierzchnictwa w PAKP - doszo do znacznego oywienia kontaktw z innymi lokalnymi Kocioami - prawosawnymi, jak rwnie do otwarcia czterech monasterw i powoania - dwch nowych diecezji oraz Prawosawnego Ordynariatu Wojska Polskiego. - Medalowe artykuy Wikipedii Aktualne Jak wybieramy najlepsze? - -Dobry artyku - - Chinchilla lanigera (Wroclaw zoo)-2.JPG - - Jeszcze w czasach prekolumbijskich Indianie wykorzystywali skrki - szynszyli do wykonywania okry wierzchnich, ktre z czasem stay si - symbolem luksusu. Na pikne futro tych zwierzt zwrcili uwag, po - podboju Ameryki Poudniowej, take Europejczycy, a w pocztku XIX wieku - szynszyle zaczy by zabijane na przemysow skal. Rabunkowa - gospodarka owiecka doprowadzia do wybicia niemal caej populacji tych - zwierzt na wolnoci. Szynszyla maa yje obecnie niemal wycznie na - terenie Parku Narodowego Szynszyli (hiszp. Las Chinchillas National - Reserve) rezerwatu pooonego w okolicy miasta Illapel w chilijskiej - prowincji Choapa. Szynszyla maa jest obecnie zaliczana przez IUCN do - gatunkw krytycznie zagroonych na wolnoci, a liczebno jej populacji - stale spada. - Zbir dobrych artykuw Jak pisa w stylu encyklopedycznym? - -Ilustracja na medal - - Aurora Borealis NO.JPG - Zorza polarna aurora borealis - Katalog medalowych ilustracji Wikipedii Jak zilustrowa artyku? - -Z ycia Wikipedii - - Aby by na bieco z tym, co si dzieje w Wikipedii, przejrzyj tablic - ogosze. - * W konkursie Wiki Lubi Zabytki pobito rekord Guinnessa przesano - ponad 160 000 fotografii. - * Zobacz co si dzieje na Planecie Wikimedia. - * Czym jest Wikipedia? Na jakich zasadach dziaa? Podstawowe - informacje o Wikipedii. - * Zapoznaj si z interaktywnym wstpem do edytowania. - * Masz pytania? Poszukaj odpowiedzi w FAQ. Zobacz te indeks - przydatnych stron. - - PL Wiki copyleft ikona.svg Prawa autorskie - PL Wiki NPOV ikona.svg Neutralny punkt widzenia - PL Wiki weryfikowalnosc ikona.svg Bez twrczoci wasnej - oraz Weryfikowalno - PL Wiki CWNJ ikona.svg Zobacz, czym Wikipedia nie jest - PL Wiki pomoc ikona.svg Wikietykieta - PL Wiki kopiowanie ikona.svg Jak legalnie kopiowa treci z - Wikipedii - - -Siostrzane projekty Wikipedii - - Commons - - Commons - Repozytorium mediw - Wikisownik - - Wikisownik - Wielojzyczny sownik - Wikispecies - - Wikispecies - Katalog gatunkw - Wikirda - - Wikirda - Dokumenty rdowe - Wikicytaty - - Wikicytaty - Kolekcja cytatw - Wikiversytet - - Wikiwersytet - Wsplnota wolnej wiedzy - Wikibooks - - Wikibooks - Wolne podrczniki - Wikinews - - Wikinews - Serwis informacyjny - Meta-Wiki - - Meta-Wiki - Koordynacja projektw - - Prawn i techniczn odpowiedzialno za Wikipedi ponosi Fundacja - Wikimedia. - - Partnerem Fundacji jest Stowarzyszenie Wikimedia Polska, zajmujce si - promocj Wikipedii w Polsce. - - Nowi uytkownicy mog znale pomoc u innych czonkw spoecznoci - m.in. u przewodnikw i administratorw. Zobacz te inne formy kontaktu - z Wikipedi. - rdo - http://pl.wikipedia.org/w/index.php?title=Strona_g%C5%82%C3%B3wna&oldi - d=28178940 - Kategoria: - * Strona gwna - -Osobiste - - * Logowanie i rejestracja - -Przestrzenie nazw - - * Strona gwna - * Dyskusja - -Warianty - -Widok - - * Czytaj - * Tekst rdowy - * Historia i autorzy - -Dziaania - -Szukaj - - ____________________ (Submit) Szukaj - -Nawigacja - - * Strona gwna - * Kategorie artykuw - * Najlepsze hasa - * Losuj artyku - -Dla czytelnikw - - * Zgo bd - * Zgo bd w pliku - * Czste pytania (FAQ) - * Kontakt - * Wykluczenie odpowiedzialnoci - * Wspom Wikipedi - -Dla wikipedystw - - * Ostatnie zmiany - * Zasady edytowania - * Pomoc - * Portal wikipedystw - * Ogoszenia - -Drukuj lub eksportuj - - * Utwrz ksik - * Pobierz jako PDF - * Wersja do druku - -Narzdzia - - * Linkujce - * Zmiany w dolinkowanych - * Strony specjalne - * Link do tej wersji - * Cytowanie tego artykuu - -W innych jzykach - * T stron ostatnio zmodyfikowano 23:44, 29 wrz 2011. - * Tekst udostpniany na licencji Creative Commons: uznanie autorstwa, - na tych samych warunkach, z moliwoci obowizywania dodatkowych - ogranicze. Zobacz szczegowe informacje o warunkach korzystania. - - * Zasady zachowania poufnoci - * O Wikipedii - * Korzystasz z Wikipedii tylko na wasn odpowiedzialno - * Wersja na komrk - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/pl/UTF-8/wikitop_pl_UTF-8.txt b/src/tests/testdata/pl/UTF-8/wikitop_pl_UTF-8.txt deleted file mode 100644 index 4707c99..0000000 --- a/src/tests/testdata/pl/UTF-8/wikitop_pl_UTF-8.txt +++ /dev/null @@ -1,271 +0,0 @@ -Strona główna - - Z Wikipedii, wolnej encyklopedii - Skocz do: nawigacji, szukaj - - Nauki ścisłe i przyrodnicze • Nauki społeczne i humanistyczne • - Technika • Geografia - Historia • Kultura • Społeczeństwo • Sport • Religie • Polska - Więcej portali • Indeks haseł • Przegląd kategorii - - Witaj w Wikipedii wolnej encyklopedii, którą każdy może redagować. - - W polskiej wersji mamy 837 872 hasła. - Gromadzimy i udostępniamy wiedzę. - Dołącz do nas! - -Czy wiesz... - - Z nowych artykułów w Wikipedii: - Slushko Palace.jpg - - …na jakim półwyspie stanął wileński Pałac Słuszków? - - …dlaczego grożono śmiercią amerykańskiej socjolożce Idzie - Wells-Barnett? - Antonowka 2.jpg - - …jak zakończyła się obrona Antonówki, ośrodka polskiej samoobrony na - Wołyniu? - - …jakim wydarzeniom w 1920 roku poświęcony był kalendarzyk - ekspresjonistyczny? - - …w których obozach koncentracyjnych był więziony Teodor Hilary - Drapiewski? - Zaproponuj artykuł do ekspozycji • Jak napisać nowe hasło? - -Wydarzenia - - Głód w Rogu Afryki • Puchar świata w rugby - __________________________________________________________________ - - Muammar al-Kaddafi, 2 lutego 2009 - * W okolicach Syrty zginął były przywódca Libii Muammar al-Kaddafi - (na zdjęciu). (20 października) - - * W wyniku negocjacji Izraela z władzami Autonomii Palestyńskiej, - Gilad Szalit został uwolniony przez Hamas w zamian za 1027 - Palestyńczyków. (18 października) - - * W trakcie wyścigu Indy Racing League doszło do karambolu, w którym - zginął Dan Wheldon. (16 października) - - * W Moskwie odsłonięto pierwszy w Rosji pomnik Jana Pawła II. Ma - ponad 1,8 m wysokości i waży dwie tony. (14 października) - - * Julia Tymoszenko została uznana za winną nadużyć i skazana na - siedem lat więzienia. (11 października) - - Więcej: w Wikipedii, w Wikinews - -Rocznice - - 21 października: imieniny obchodzą Celina, Jakub, Urszula - Willy Brandt - * 1824 – Joseph Aspdin opatentował cement portlandzki - * 1833 – urodził się Alfred Nobel, szwedzki chemik, wynalazca i - filantrop, fundator Nagrody Nobla - * 1929 – urodziła się Ursula K. Le Guin, amerykańska pisarka - fantastyki - * 1956 – I sekretarzem PZPR w miejsce Edwarda Ochaba został Władysław - Gomułka - * 1963 – w obławie SB i ZOMO zginął Józef Franczak ps. Lalek, ostatni - żołnierz polskiego podziemia - * 1969 – zmarł Wacław Sierpiński, polski matematyk - * 1969 – Willy Brandt (na zdjęciu), przywódca SPD, został kanclerzem - Niemieckiej Republiki Federalnej - - 20 października • Kalendarium dzień po dniu • 22 października - -Artykuł na medal - - Bazyli doroszkiewicz.jpg - Bazyli, imię świeckie: Włodzimierz Doroszkiewicz (ur. 15 marca 1914 w - Cisach w Prusach Wschodnich, zm. 11 lutego 1998 w Warszawie) – piąty - metropolita warszawski i całej Polski, zwierzchnik Polskiego - Autokefalicznego Kościoła Prawosławnego od 1970 do śmierci w 1998. W - 1938 przyjął święcenia kapłańskie. Do 1960 prowadził pracę - duszpasterską w różnych parafiach na Białostocczyźnie oraz - Grodzieńszczyźnie. W grudniu 1959 złożył wieczyste śluby zakonne, - przyjmując imię Bazyli. W 1970 objął urząd metropolity warszawskiego i - całej Polski. W okresie pełnienia przez niego zwierzchnictwa w PAKP - doszło do znacznego ożywienia kontaktów z innymi lokalnymi Kościołami - prawosławnymi, jak również do otwarcia czterech monasterów i powołania - dwóch nowych diecezji oraz Prawosławnego Ordynariatu Wojska Polskiego. - Medalowe artykuły Wikipedii • Aktualne • Jak wybieramy najlepsze? - -Dobry artykuł - - Chinchilla lanigera (Wroclaw zoo)-2.JPG - - Jeszcze w czasach prekolumbijskich Indianie wykorzystywali skórki - szynszyli do wykonywania okryć wierzchnich, które z czasem stały się - symbolem luksusu. Na piękne futro tych zwierząt zwrócili uwagę, po - podboju Ameryki Południowej, także Europejczycy, a w początku XIX wieku - szynszyle zaczęły być zabijane na przemysłową skalę. Rabunkowa - gospodarka łowiecka doprowadziła do wybicia niemal całej populacji tych - zwierząt na wolności. Szynszyla mała żyje obecnie niemal wyłącznie na - terenie Parku Narodowego Szynszyli (hiszp. Las Chinchillas National - Reserve) – rezerwatu położonego w okolicy miasta Illapel w chilijskiej - prowincji Choapa. Szynszyla mała jest obecnie zaliczana przez IUCN do - gatunków krytycznie zagrożonych na wolności, a liczebność jej populacji - stale spada. - Zbiór dobrych artykułów • Jak pisać w stylu encyklopedycznym? - -Ilustracja na medal - - Aurora Borealis NO.JPG - Zorza polarna aurora borealis - Katalog medalowych ilustracji Wikipedii • Jak zilustrować artykuł? - -Z życia Wikipedii - - Aby być na bieżąco z tym, co się dzieje w Wikipedii, przejrzyj tablicę - ogłoszeń. - * W konkursie Wiki Lubi Zabytki pobito rekord Guinnessa – przesłano - ponad 160 000 fotografii. - * Zobacz co się dzieje na Planecie Wikimedia. - * Czym jest Wikipedia? Na jakich zasadach działa? Podstawowe - informacje o Wikipedii. - * Zapoznaj się z interaktywnym wstępem do edytowania. - * Masz pytania? Poszukaj odpowiedzi w FAQ. Zobacz też indeks - przydatnych stron. - - PL Wiki copyleft ikona.svg Prawa autorskie - PL Wiki NPOV ikona.svg Neutralny punkt widzenia - PL Wiki weryfikowalnosc ikona.svg Bez twórczości własnej - oraz Weryfikowalność - PL Wiki CWNJ ikona.svg Zobacz, czym Wikipedia nie jest - PL Wiki pomoc ikona.svg Wikietykieta - PL Wiki kopiowanie ikona.svg Jak legalnie kopiować treści z - Wikipedii - - -Siostrzane projekty Wikipedii - - Commons - - Commons - Repozytorium mediów - Wikisłownik - - Wikisłownik - Wielojęzyczny słownik - Wikispecies - - Wikispecies - Katalog gatunków - Wikiźródła - - Wikiźródła - Dokumenty źródłowe - Wikicytaty - - Wikicytaty - Kolekcja cytatów - Wikiversytet - - Wikiwersytet - Wspólnota wolnej wiedzy - Wikibooks - - Wikibooks - Wolne podręczniki - Wikinews - - Wikinews - Serwis informacyjny - Meta-Wiki - - Meta-Wiki - Koordynacja projektów - - Prawną i techniczną odpowiedzialność za Wikipedię ponosi Fundacja - Wikimedia. - - Partnerem Fundacji jest Stowarzyszenie Wikimedia Polska, zajmujące się - promocją Wikipedii w Polsce. - - Nowi użytkownicy mogą znaleźć pomoc u innych członków społeczności - m.in. u przewodników i administratorów. Zobacz też inne formy kontaktu - z Wikipedią. - Źródło - „http://pl.wikipedia.org/w/index.php?title=Strona_g%C5%82%C3%B3wna&oldi - d=28178940” - Kategoria: - * Strona główna - -Osobiste - - * Logowanie i rejestracja - -Przestrzenie nazw - - * Strona główna - * Dyskusja - -Warianty - -Widok - - * Czytaj - * Tekst źródłowy - * Historia i autorzy - -Działania - -Szukaj - - ____________________ (Submit) Szukaj - -Nawigacja - - * Strona główna - * Kategorie artykułów - * Najlepsze hasła - * Losuj artykuł - -Dla czytelników - - * Zgłoś błąd - * Zgłoś błąd w pliku - * Częste pytania (FAQ) - * Kontakt - * Wykluczenie odpowiedzialności - * Wspomóż Wikipedię - -Dla wikipedystów - - * Ostatnie zmiany - * Zasady edytowania - * Pomoc - * Portal wikipedystów - * Ogłoszenia - -Drukuj lub eksportuj - - * Utwórz książkę - * Pobierz jako PDF - * Wersja do druku - -Narzędzia - - * Linkujące - * Zmiany w dolinkowanych - * Strony specjalne - * Link do tej wersji - * Cytowanie tego artykułu - - * Tę stronę ostatnio zmodyfikowano 23:44, 29 wrz 2011. - * Tekst udostępniany na licencji Creative Commons: uznanie autorstwa, - na tych samych warunkach, z możliwością obowiązywania dodatkowych - ograniczeń. Zobacz szczegółowe informacje o warunkach korzystania. - - * Zasady zachowania poufności - * O Wikipedii - * Korzystasz z Wikipedii tylko na własną odpowiedzialność - * Wersja na komórkę - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/pl/ibm852.txt b/src/tests/testdata/pl/ibm852.txt new file mode 100644 index 0000000..e420950 --- /dev/null +++ b/src/tests/testdata/pl/ibm852.txt @@ -0,0 +1,3 @@ +Zofia (Sonka) Holszaska herbu Hippocentaurus (ur. ok. 1405, zm. 21 wrzenia 1461 w Krakowie) +ksiniczka litewska, krlowa Polski, od 1422 roku czwarta i ostatnia ona Wadysawa II +Jagiey. diff --git a/src/tests/testdata/pl/iso-8859-13.txt b/src/tests/testdata/pl/iso-8859-13.txt new file mode 100644 index 0000000..6bafbc1 --- /dev/null +++ b/src/tests/testdata/pl/iso-8859-13.txt @@ -0,0 +1,3 @@ +Zofia (Sonka) Holszaska herbu Hippocentaurus (ur. ok. 1405, zm. 21 wrzenia 1461 w Krakowie) +ksiniczka litewska, krlowa Polski, od 1422 roku czwarta i ostatnia ona Wadysawa II +Jagiey. diff --git a/src/tests/testdata/pl/iso-8859-16.txt b/src/tests/testdata/pl/iso-8859-16.txt new file mode 100644 index 0000000..abe9607 --- /dev/null +++ b/src/tests/testdata/pl/iso-8859-16.txt @@ -0,0 +1,3 @@ +Zofia (Sonka) Holszaska herbu Hippocentaurus (ur. ok. 1405, zm. 21 wrzenia 1461 w Krakowie) +ksiniczka litewska, krlowa Polski, od 1422 roku czwarta i ostatnia ona Wadysawa II +Jagiey. diff --git a/src/tests/testdata/pl/iso-8859-2.txt b/src/tests/testdata/pl/iso-8859-2.txt new file mode 100644 index 0000000..8ff7d6d --- /dev/null +++ b/src/tests/testdata/pl/iso-8859-2.txt @@ -0,0 +1,3 @@ +Zofia (Sonka) Holszaska herbu Hippocentaurus (ur. ok. 1405, zm. 21 wrzenia 1461 w Krakowie) +ksiniczka litewska, krlowa Polski, od 1422 roku czwarta i ostatnia ona Wadysawa II +Jagiey. diff --git a/src/tests/testdata/pl/maccentraleurope.txt b/src/tests/testdata/pl/maccentraleurope.txt new file mode 100644 index 0000000..48c5901 --- /dev/null +++ b/src/tests/testdata/pl/maccentraleurope.txt @@ -0,0 +1,3 @@ +Zofia (Sonka) Holszaska herbu Hippocentaurus (ur. ok. 1405, zm. 21 wrzenia 1461 w Krakowie) +ksiniczka litewska, krlowa Polski, od 1422 roku czwarta i ostatnia ona Wadysawa II +Jagiey. diff --git a/src/tests/testdata/pl/utf-8.txt b/src/tests/testdata/pl/utf-8.txt new file mode 100644 index 0000000..8b7e938 --- /dev/null +++ b/src/tests/testdata/pl/utf-8.txt @@ -0,0 +1,3 @@ +Zofia (Sonka) Holszańska herbu Hippocentaurus (ur. ok. 1405, zm. 21 września 1461 w Krakowie) +księżniczka litewska, królowa Polski, od 1422 roku czwarta i ostatnia żona Władysława II +Jagiełły. diff --git a/src/tests/testdata/pl/windows-1250.txt b/src/tests/testdata/pl/windows-1250.txt new file mode 100644 index 0000000..c739798 --- /dev/null +++ b/src/tests/testdata/pl/windows-1250.txt @@ -0,0 +1,3 @@ +Zofia (Sonka) Holszaska herbu Hippocentaurus (ur. ok. 1405, zm. 21 wrzenia 1461 w Krakowie) +ksiniczka litewska, krlowa Polski, od 1422 roku czwarta i ostatnia ona Wadysawa II +Jagiey. diff --git a/src/tests/testdata/pt/UTF-8/wikitop_pt_UTF-8.txt b/src/tests/testdata/pt/UTF-8/wikitop_pt_UTF-8.txt deleted file mode 100644 index a6837f2..0000000 --- a/src/tests/testdata/pt/UTF-8/wikitop_pt_UTF-8.txt +++ /dev/null @@ -1,309 +0,0 @@ -Wikipédia:Página principal - - Origem: Wikipédia, a enciclopédia livre. - Ir para: navegação, pesquisa - - Bem-vindo(a) à Wikipédia, - a enciclopédia livre que todos podem editar. 13h47min (UTC); - sexta-feira, 21 de outubro de 2011 - 701 502 artigos (193 bons e 405 destacados) em português. - Portais · Livros · Índice geral · Ajuda · Estatísticas - - Guia · Perguntas frequentes · Comunidade · Políticas da Wikipédia · - Doações · WAP · Acessibilidade · Contato - - Predefinição:Em destaque Artigo em destaque - Abertura Jogos Panamericanos 2 13072007.jpg - - A cerimônia de abertura dos Jogos Pan-Americanos de 2007 marcou o - início oficial dos XV Jogos Pan-Americanos, ocorridos na cidade do Rio - de Janeiro de 12 a 29 de julho de 2007. Sua realização ocorreu no dia - 13 de julho às 18 horas (UTC−3) no Estádio do Maracanã, localizado no - bairro homônimo, e teve duração de três horas e meia. - Produzido pelo americano Scott Givens, que já havia trabalhado em - diversas cerimônias de abertura de Jogos Pan-Americanos e Jogos - Olímpicos, o evento teve como diretores artísticos a carnavalesca Rosa - Magalhães e o designer Luiz Stein. (leia mais...) - Wikipedia:Efemérides/21 de outubro Neste dia… - Pelé - - 21 de outubro: Dia do Ecumenismo. - * 1187 - É eleito o Papa Gregório VIII. - * 1838 - Dom Pedro II do Brasil funda o Instituto Histórico e - Geográfico Brasileiro. - * 1895 - Forças japonesas invadem a capital de Taiwan, dando início - ao domínio japonês naquele país. - - Nasceram neste dia… - * 1833 - Alfred Nobel , inventor sueco fundador dos Prémios Nobel (m. - 1896). - * 1917 - Dizzy Gillespie, trompetista, cantor e compositor de jazz - estadunidense (m. 1993). - * 1940 - Pelé (na foto), futebolista brasileiro. - - Morreram neste dia… - * 1422 - Carlos VI de Valois, Rei de França (n. 1368). - * 1805 - Horatio Nelson, almirante britânico (n. 1758). - * 1984 - François Truffaut, director de cinema francês. (n. 1932). - - Outros dias: 19 · 20 · 21 · 22 · 23 | ver todos… - - Predefinição:Sabia que Sabia que… - Contactlenzen Confortissimo.JPG - * ... cerca de 125 milhões de pessoas em todo mundo usam lentes de - contato (foto), quase 2% da população mundial? - * ... o AVE Mizar foi um protótipo de carro voador criado no começo - da década de 1970 a partir da junção de um Cessna Skymaster com um - Ford Pinto? - * ... é possível velejar contra o vento, e que esta técnica chama-se - bolina? - * ... a bandeira dos Estados Unidos colocada na Lua pela Apollo 11 - foi feita por uma portuguesa chamada Maria Isilda Ribeiro? - - - Predefinição:Eventos atuais Eventos recentes - Muammar al-Gaddafi at the AU summit-LR.jpg - * ETA anuncia fim de atividades terroristas após 51 anos e 800 - mortos. - * Muammar al-Gaddafi (foto), ex-ditador da Líbia, morre em combate em - Sirte. - * Gilad Shalit, soldado de Israel, libertado após mais de cinco anos - de cativeiro em troca da libertação de 1027 prisioneiros - palestinianos. - * Realizada a cerimônia de abertura dos Jogos Pan-Americanos de 2011, - em Guadalajara, no México. - * Prémio de Ciências Económicas atribuído a Christopher Sims e Thomas - Sargent pelo trabalho sobre relações causais entre política - económica e variáveis macroeconómicas. - - Mortes recentes – Mais eventos atuais… - __________________________________________________________________ - - Veja mais no Wikinotícias - Wikipedia:Imagem em destaque/21 de outubro de 2011 Imagem do dia - Inside the Tarawera rift.jpg - - Monte Tarawera, Nova Zelândia. - - Sobre a Wikipédia - - Importante! — Princípio da imparcialidade · Versões do português · - Direitos de autor · Normas de conduta · Coisas a não fazer - __________________________________________________________________ - - Escrevendo artigos — O que é um wiki? · Livro de estilo · Como - contribuir · Tutorial · Recursos no domínio público · Conteúdo - destacado - __________________________________________________________________ - - Ajude a Wikipédia — Artigos com problemas · Páginas a traduzir · - Artigos à espera de autor · Estaleiro · Donativos - __________________________________________________________________ - - Sobre a Wikipédia — Esplanada · FAQ · Contato · Wikimedia · Software · - Estatísticas · Consulta e reprodução · Decisões da comunidade · Informe - um erro - __________________________________________________________________ - - Domínios — Wikipédia · MediaWiki · Predefinição · Anexo · Ajuda - - - - Interesse geral - - Filosofia · Matemática · Epistemologia - Ciências humanas e sociais - - Antropologia · Arqueologia · Direito · Economia · Geografia · História - · Linguística · Psicologia · Sociologia - Arte - - Arquitetura e urbanismo · Cinema · Dança · Escultura · Fotografia · - Literatura · Música · Música Clássica · Pintura · Teatro - Política - - Diplomacia · Eleições · Movimentos · Partidos · Políticos · Regimes - políticos - Biografias - - Artistas · Chefes de Estado · Cientistas · Escritores · Filósofos · - Políticos · Religiosos - Saúde - - Doenças · Drogas · Enfermagem · Farmácia · Medicina · Nutrição · - Odontologia · Saúde sexual - Ciências exatas, naturais e tecnologia - - Astronomia · Biologia · Ciências da Terra · Eletrônica · Física · - Informática · Meteorologia · Química - Sociedade - - Comportamento · Culinária · Cultura · Diversão · Esporte/desporto · - Religião · Turismo - Wikipédia em outros idiomas - - Wikipédias com mais de 1 000 000 de artigos: - Deutsch (alemão) · English (inglês) · Français (francês) - - Wikipédias com mais de 500 000 artigos: - Español (espanhol) · Italiano · 日本語 (japonês) · Nederlands -(neerlandês) · Polski (polaco/polonês) · Português · Русский (russo) - - Wikipédias com mais de 100 000 artigos: -العربية (árabe) · Euskara (basco) · Български (búlgaro) · Català -(catalão) · 한국어 (coreano) · 中文 (chinês) · Hrvatski (croata) · Dansk - (dinamarquês) · Slovencina (eslovaco) · Slovenščina (esloveno) · -Esperanto · Suomi (finlandês) · עברית (hebraico) · हिन्दी (hindi) · - Magyar (húngaro) · Bahasa Indonesia (indonésio) · Lietuviu (lituano) · -Bahasa Melayu (malaio) · Norsk (norueguês) · فارسی (persa) · Română - (romeno) · Српски / Srpski (sérvio) · Svenska (sueco) · Česká -(tcheco/checo) · Türkçe (turco) · Українська (ucraniano) · Tiếng Việt - (vietnamita) · Volapük (volapuque) · Winaray - - Lista completa · Coordenação multilíngue · Começar uma nova Wikipédia - -Projetos Wikimedia - - A Wikipédia existe graças à Wikimedia Foundation, entidade sem fins - lucrativos que gere projetos em diversos idiomas e de conteúdo livre: - - - Wikiquote Wikiquote - Coletânea de citações - Wikcionário Wikcionário - Dicionário de várias línguas - Wikispecies Wikispecies - Diretório de espécies - - Wikinotícias Wikinotícias - Fonte de notícias livres - Wikisource Wikisource - Documentos originais livres - Commons Wikimedia Commons - Imagens, sons e vídeos - - Wikiversidade Wikiversidade - Centro ilimitado do aprender - Wikibooks Wikilivros - Livros e manuais livres - Meta-Wiki Meta-Wiki - Coordenação dos projetos - - Obtida de - "http://pt.wikipedia.org/w/index.php?title=Wikip%C3%A9dia:P%C3%A1gina_p - rincipal&oldid=24857386" - -Ferramentas pessoais - - * Entrar / criar conta - -Espaços nominais - - * Página do projeto - * Discussão - -Variantes - -Vistas - - * Ler - * Ver conteúdo - * Ver histórico - -Ações - -Busca - - ____________________ (Submit) Pesquisar - -Navegação - - * Página principal - * Conteúdo destacado - * Eventos atuais - * Esplanada - * Página aleatória - * Portais - * Informar um erro - -Colaboração - - * Boas-vindas - * Ajuda - * Página de testes - * Portal comunitário - * Mudanças recentes - * Estaleiro - * Criar página - * Páginas novas - * Contato - * Donativos - -Imprimir/exportar - - * Criar um livro - * Descarregar como PDF - * Versão para impressão - -Ferramentas - - * Páginas afluentes - * Alterações relacionadas - * Carregar ficheiro - * Páginas especiais - * Ligação permanente - -Noutras línguas - - * العربية - * Bahasa Indonesia - * Български - * Català - * Česky - * Dansk - * Deutsch - * English - * Eesti - * Ελληνικά - * Español - * Esperanto - * Euskara - * فارسی - * Français - * Galego - * עברית - * Italiano - * 한국어 - * Lietuvių - * Magyar - * Mirandés - * Nederlands - * 日本語 - * ‪Norsk (bokmål)‬ - * ‪Norsk (nynorsk)‬ - * Polski - * Română - * Русский - * Slovenčina - * Slovenščina - * Српски / Srpski - * Suomi - * Svenska - * Türkçe - * Tiếng Việt - * 中文 - - * Esta página foi modificada pela última vez à(s) 21h17min de 17 de - abril de 2011. - * Este texto é disponibilizado nos termos da licença Atribuição - - Partilha nos Mesmos Termos 3.0 Não Adaptada (CC BY-SA 3.0); pode - estar sujeito a condições adicionais. Consulte as condições de uso - para mais detalhes. - - * Política de privacidade - * Sobre a Wikipédia - * Avisos gerais - * Versão móvel - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/pt/WINDOWS-1252/wikitop_pt_WINDOWS-1252.txt b/src/tests/testdata/pt/WINDOWS-1252/wikitop_pt_WINDOWS-1252.txt deleted file mode 100644 index 3bc0e3e..0000000 --- a/src/tests/testdata/pt/WINDOWS-1252/wikitop_pt_WINDOWS-1252.txt +++ /dev/null @@ -1,309 +0,0 @@ -Wikipdia:Pgina principal - - Origem: Wikipdia, a enciclopdia livre. - Ir para: navegao, pesquisa - - Bem-vindo(a) Wikipdia, - a enciclopdia livre que todos podem editar. 13h47min (UTC); - sexta-feira, 21 de outubro de 2011 - 701 502 artigos (193 bons e 405 destacados) em portugus. - Portais Livros ndice geral Ajuda Estatsticas - - Guia Perguntas frequentes Comunidade Polticas da Wikipdia - Doaes WAP Acessibilidade Contato - - Predefinio:Em destaque Artigo em destaque - Abertura Jogos Panamericanos 2 13072007.jpg - - A cerimnia de abertura dos Jogos Pan-Americanos de 2007 marcou o - incio oficial dos XV Jogos Pan-Americanos, ocorridos na cidade do Rio - de Janeiro de 12 a 29 de julho de 2007. Sua realizao ocorreu no dia - 13 de julho s 18 horas (UTC3) no Estdio do Maracan, localizado no - bairro homnimo, e teve durao de trs horas e meia. - Produzido pelo americano Scott Givens, que j havia trabalhado em - diversas cerimnias de abertura de Jogos Pan-Americanos e Jogos - Olmpicos, o evento teve como diretores artsticos a carnavalesca Rosa - Magalhes e o designer Luiz Stein. (leia mais...) - Wikipedia:Efemrides/21 de outubro Neste dia - Pel - - 21 de outubro: Dia do Ecumenismo. - * 1187 - eleito o Papa Gregrio VIII. - * 1838 - Dom Pedro II do Brasil funda o Instituto Histrico e - Geogrfico Brasileiro. - * 1895 - Foras japonesas invadem a capital de Taiwan, dando incio - ao domnio japons naquele pas. - - Nasceram neste dia - * 1833 - Alfred Nobel , inventor sueco fundador dos Prmios Nobel (m. - 1896). - * 1917 - Dizzy Gillespie, trompetista, cantor e compositor de jazz - estadunidense (m. 1993). - * 1940 - Pel (na foto), futebolista brasileiro. - - Morreram neste dia - * 1422 - Carlos VI de Valois, Rei de Frana (n. 1368). - * 1805 - Horatio Nelson, almirante britnico (n. 1758). - * 1984 - Franois Truffaut, director de cinema francs. (n. 1932). - - Outros dias: 19 20 21 22 23 | ver todos - - Predefinio:Sabia que Sabia que - Contactlenzen Confortissimo.JPG - * ... cerca de 125 milhes de pessoas em todo mundo usam lentes de - contato (foto), quase 2% da populao mundial? - * ... o AVE Mizar foi um prottipo de carro voador criado no comeo - da dcada de 1970 a partir da juno de um Cessna Skymaster com um - Ford Pinto? - * ... possvel velejar contra o vento, e que esta tcnica chama-se - bolina? - * ... a bandeira dos Estados Unidos colocada na Lua pela Apollo 11 - foi feita por uma portuguesa chamada Maria Isilda Ribeiro? - - - Predefinio:Eventos atuais Eventos recentes - Muammar al-Gaddafi at the AU summit-LR.jpg - * ETA anuncia fim de atividades terroristas aps 51 anos e 800 - mortos. - * Muammar al-Gaddafi (foto), ex-ditador da Lbia, morre em combate em - Sirte. - * Gilad Shalit, soldado de Israel, libertado aps mais de cinco anos - de cativeiro em troca da libertao de 1027 prisioneiros - palestinianos. - * Realizada a cerimnia de abertura dos Jogos Pan-Americanos de 2011, - em Guadalajara, no Mxico. - * Prmio de Cincias Econmicas atribudo a Christopher Sims e Thomas - Sargent pelo trabalho sobre relaes causais entre poltica - econmica e variveis macroeconmicas. - - Mortes recentes Mais eventos atuais - __________________________________________________________________ - - Veja mais no Wikinotcias - Wikipedia:Imagem em destaque/21 de outubro de 2011 Imagem do dia - Inside the Tarawera rift.jpg - - Monte Tarawera, Nova Zelndia. - - Sobre a Wikipdia - - Importante! Princpio da imparcialidade Verses do portugus - Direitos de autor Normas de conduta Coisas a no fazer - __________________________________________________________________ - - Escrevendo artigos O que um wiki? Livro de estilo Como - contribuir Tutorial Recursos no domnio pblico Contedo - destacado - __________________________________________________________________ - - Ajude a Wikipdia Artigos com problemas Pginas a traduzir - Artigos espera de autor Estaleiro Donativos - __________________________________________________________________ - - Sobre a Wikipdia Esplanada FAQ Contato Wikimedia Software - Estatsticas Consulta e reproduo Decises da comunidade Informe - um erro - __________________________________________________________________ - - Domnios Wikipdia MediaWiki Predefinio Anexo Ajuda - - - - Interesse geral - - Filosofia Matemtica Epistemologia - Cincias humanas e sociais - - Antropologia Arqueologia Direito Economia Geografia Histria - Lingustica Psicologia Sociologia - Arte - - Arquitetura e urbanismo Cinema Dana Escultura Fotografia - Literatura Msica Msica Clssica Pintura Teatro - Poltica - - Diplomacia Eleies Movimentos Partidos Polticos Regimes - polticos - Biografias - - Artistas Chefes de Estado Cientistas Escritores Filsofos - Polticos Religiosos - Sade - - Doenas Drogas Enfermagem Farmcia Medicina Nutrio - Odontologia Sade sexual - Cincias exatas, naturais e tecnologia - - Astronomia Biologia Cincias da Terra Eletrnica Fsica - Informtica Meteorologia Qumica - Sociedade - - Comportamento Culinria Cultura Diverso Esporte/desporto - Religio Turismo - Wikipdia em outros idiomas - - Wikipdias com mais de 1 000 000 de artigos: - Deutsch (alemo) English (ingls) Franais (francs) - - Wikipdias com mais de 500 000 artigos: - Espaol (espanhol) Italiano (japons) Nederlands -(neerlands) Polski (polaco/polons) Portugus (russo) - - Wikipdias com mais de 100 000 artigos: - (rabe) Euskara (basco) (blgaro) Catal -(catalo) (coreano) (chins) Hrvatski (croata) Dansk - (dinamarqus) Slovencina (eslovaco) Slovenina (esloveno) -Esperanto Suomi (finlands) (hebraico) (hindi) - Magyar (hngaro) Bahasa Indonesia (indonsio) Lietuviu (lituano) -Bahasa Melayu (malaio) Norsk (noruegus) (persa) Romn - (romeno) / Srpski (srvio) Svenska (sueco) esk -(tcheco/checo) Trke (turco) (ucraniano) Ting Vit - (vietnamita) Volapk (volapuque) Winaray - - Lista completa Coordenao multilngue Comear uma nova Wikipdia - -Projetos Wikimedia - - A Wikipdia existe graas Wikimedia Foundation, entidade sem fins - lucrativos que gere projetos em diversos idiomas e de contedo livre: - - - Wikiquote Wikiquote - Coletnea de citaes - Wikcionrio Wikcionrio - Dicionrio de vrias lnguas - Wikispecies Wikispecies - Diretrio de espcies - - Wikinotcias Wikinotcias - Fonte de notcias livres - Wikisource Wikisource - Documentos originais livres - Commons Wikimedia Commons - Imagens, sons e vdeos - - Wikiversidade Wikiversidade - Centro ilimitado do aprender - Wikibooks Wikilivros - Livros e manuais livres - Meta-Wiki Meta-Wiki - Coordenao dos projetos - - Obtida de - "http://pt.wikipedia.org/w/index.php?title=Wikip%C3%A9dia:P%C3%A1gina_p - rincipal&oldid=24857386" - -Ferramentas pessoais - - * Entrar / criar conta - -Espaos nominais - - * Pgina do projeto - * Discusso - -Variantes - -Vistas - - * Ler - * Ver contedo - * Ver histrico - -Aes - -Busca - - ____________________ (Submit) Pesquisar - -Navegao - - * Pgina principal - * Contedo destacado - * Eventos atuais - * Esplanada - * Pgina aleatria - * Portais - * Informar um erro - -Colaborao - - * Boas-vindas - * Ajuda - * Pgina de testes - * Portal comunitrio - * Mudanas recentes - * Estaleiro - * Criar pgina - * Pginas novas - * Contato - * Donativos - -Imprimir/exportar - - * Criar um livro - * Descarregar como PDF - * Verso para impresso - -Ferramentas - - * Pginas afluentes - * Alteraes relacionadas - * Carregar ficheiro - * Pginas especiais - * Ligao permanente - -Noutras lnguas - - * - * Bahasa Indonesia - * - * Catal - * esky - * Dansk - * Deutsch - * English - * Eesti - * - * Espaol - * Esperanto - * Euskara - * - * Franais - * Galego - * - * Italiano - * - * Lietuvi - * Magyar - * Mirands - * Nederlands - * - * Norsk (bokml) - * Norsk (nynorsk) - * Polski - * Romn - * - * Slovenina - * Slovenina - * / Srpski - * Suomi - * Svenska - * Trke - * Ting Vit - * - - * Esta pgina foi modificada pela ltima vez (s) 21h17min de 17 de - abril de 2011. - * Este texto disponibilizado nos termos da licena Atribuio - - Partilha nos Mesmos Termos 3.0 No Adaptada (CC BY-SA 3.0); pode - estar sujeito a condies adicionais. Consulte as condies de uso - para mais detalhes. - - * Poltica de privacidade - * Sobre a Wikipdia - * Avisos gerais - * Verso mvel - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/pt/iso-8859-1.txt b/src/tests/testdata/pt/iso-8859-1.txt new file mode 100644 index 0000000..ec6bdda --- /dev/null +++ b/src/tests/testdata/pt/iso-8859-1.txt @@ -0,0 +1,6 @@ +Albertossauro (Albertosaurus sp., que significa "lagarto de Alberta" no Canad), +foi um gnero de dinossauro carnvoro e bpede presente no fim do perodo +Cretceo. Media cerca de 8 a 9 metros de comprimento, 3 metros de altura e +pesava menos de 2 toneladas. O Albertossauro viveu na Amrica do Norte e foi +descoberto no ano de 1884 por Joseph Burr Tyrrell em Alberta, no Canad, local +ao qual deve seu nome. diff --git a/src/tests/testdata/pt/utf-8.txt b/src/tests/testdata/pt/utf-8.txt new file mode 100644 index 0000000..1729291 --- /dev/null +++ b/src/tests/testdata/pt/utf-8.txt @@ -0,0 +1,6 @@ +Albertossauro (Albertosaurus sp., que significa "lagarto de Alberta" no Canadá), +foi um género de dinossauro carnívoro e bípede presente no fim do período +Cretáceo. Media cerca de 8 a 9 metros de comprimento, 3 metros de altura e +pesava menos de 2 toneladas. O Albertossauro viveu na América do Norte e foi +descoberto no ano de 1884 por Joseph Burr Tyrrell em Alberta, no Canadá, local +ao qual deve seu nome. diff --git a/src/tests/testdata/ro/ibm852.txt b/src/tests/testdata/ro/ibm852.txt new file mode 100644 index 0000000..634dda2 --- /dev/null +++ b/src/tests/testdata/ro/ibm852.txt @@ -0,0 +1,9 @@ +Danemarca (n danez Sunet Danmark), oficial Regatul Danemarcei (n +danez Sunet Kongeriget Danmark), este un stat suveran din +Europa de Nord, avnd si dou tri constituente de peste mri, care fac parte +integrant din regat: Insulele Feroe n Atlanticul de Nord si Groenlanda n +America de Nord. Danemarca propriu-zis[a] este cea mai de sud dintre trile +nordice, aflat la sud-vest de Suedia si la sud de Norvegia, nvecinndu-se la +sud cu Germania. Tara const dintr-o peninsul mare, Iutlanda, si mai multe +insule, dintre care cele mai mari sunt Zealand, Funen, Lolland, Falster si +Bornholm, precum si sute de insulite denumite n general ,,Arhipelagul Danez". diff --git a/src/tests/testdata/ro/iso-8859-16.txt b/src/tests/testdata/ro/iso-8859-16.txt new file mode 100644 index 0000000..29ae299 --- /dev/null +++ b/src/tests/testdata/ro/iso-8859-16.txt @@ -0,0 +1,9 @@ +Danemarca (n danez Sunet Danmark), oficial Regatul Danemarcei (n +danez Sunet Kongeriget Danmark), este un stat suveran din +Europa de Nord, avnd i dou ri constituente de peste mri, care fac parte +integrant din regat: Insulele Feroe n Atlanticul de Nord i Groenlanda n +America de Nord. Danemarca propriu-zis[a] este cea mai de sud dintre rile +nordice, aflat la sud-vest de Suedia i la sud de Norvegia, nvecinndu-se la +sud cu Germania. ara const dintr-o peninsul mare, Iutlanda, i mai multe +insule, dintre care cele mai mari sunt Zealand, Funen, Lolland, Falster i +Bornholm, precum i sute de insulie denumite n general Arhipelagul Danez. diff --git a/src/tests/testdata/ro/utf-8.txt b/src/tests/testdata/ro/utf-8.txt new file mode 100644 index 0000000..dea759e --- /dev/null +++ b/src/tests/testdata/ro/utf-8.txt @@ -0,0 +1,9 @@ +Danemarca (în daneză Sunet Danmark), oficial Regatul Danemarcei (în +daneză Sunet Kongeriget Danmark), este un stat suveran din +Europa de Nord, având și două țări constituente de peste mări, care fac parte +integrantă din regat: Insulele Feroe în Atlanticul de Nord și Groenlanda în +America de Nord. Danemarca propriu-zisă[a] este cea mai de sud dintre țările +nordice, aflată la sud-vest de Suedia și la sud de Norvegia, învecinându-se la +sud cu Germania. Țara constă dintr-o peninsulă mare, Iutlanda, și mai multe +insule, dintre care cele mai mari sunt Zealand, Funen, Lolland, Falster și +Bornholm, precum și sute de insulițe denumite în general „Arhipelagul Danez”. diff --git a/src/tests/testdata/ro/windows-1250.txt b/src/tests/testdata/ro/windows-1250.txt new file mode 100644 index 0000000..f43cb89 --- /dev/null +++ b/src/tests/testdata/ro/windows-1250.txt @@ -0,0 +1,9 @@ +Danemarca (n danez Sunet Danmark), oficial Regatul Danemarcei (n +danez Sunet Kongeriget Danmark), este un stat suveran din +Europa de Nord, avnd si dou tri constituente de peste mri, care fac parte +integrant din regat: Insulele Feroe n Atlanticul de Nord si Groenlanda n +America de Nord. Danemarca propriu-zis[a] este cea mai de sud dintre trile +nordice, aflat la sud-vest de Suedia si la sud de Norvegia, nvecinndu-se la +sud cu Germania. Tara const dintr-o peninsul mare, Iutlanda, si mai multe +insule, dintre care cele mai mari sunt Zealand, Funen, Lolland, Falster si +Bornholm, precum si sute de insulite denumite n general Arhipelagul Danez. diff --git a/src/tests/testdata/ru/IBM855/wikitop_ru_IBM855.txt b/src/tests/testdata/ru/IBM855/wikitop_ru_IBM855.txt deleted file mode 100644 index 0f768c4..0000000 --- a/src/tests/testdata/ru/IBM855/wikitop_ru_IBM855.txt +++ /dev/null @@ -1,312 +0,0 @@ - #Ʒب (ru) copyright Ʒب Atom-Ш - -РԠ Է - - Ӡᷠ Ʒب ֽ֢֦ Ԥب - 堢Ԡ Р Ԡ 19 ޢ 2011. Ҩ - Ԩ ҨԨԷ Ԡ з Р. - Ơ/ ֦֢ - 堢Ԡ Р Ԡ 19 ޢ 2011. Ҩ - Ԩ ҨԨԷ Ԡ з Р. - ݨᨽ : Ԡ뷬, ַ - - - ֢ Ʒب, - - ֢֦ Ԥب, ᨦ Ơ. - 䨽 Ʒب 779 821 Ԡ ƨ. - - ֬ᠪ - - - Ơ - - ՠƠ - ע - ݨԠз - з - - - 樵ԷƠ - ֪ - - ( Է) Ơ Ҡ Ơ - Ʒب - -Ԡ - - 䵨Ҡ ᨦШԷ II Ƥ - - Ʒ ب Ԅ֨ ԠԷ - ַ 3 (16) 1907 ֦ 㷷 ֬ Ơ II - 禠ֽ , 馠 ҨԨԷ ֽ - . - - ֽ Ơ II Р Ԩ ԠР - ֨ ֦뷨 Ҩ , Рި - -ҷԷ . . ط, ֽ, Ԡ ֽ - з ᨦз ᠽԷ Ш ؠ巽 (֤-, - ֤з-МԨ, Ԡ֦ ֤з) Ơ Է - Ʒ. II Ҡ, 20 1907 ֦, ҨР Ԩ ҨԨ - 󷤷֨ Ԡ֨Է, Ԩ Ԡ I Ҡ. Ҡ Р - ԦԤ ԨԷ 㨵 ֨ , - ֨, ᨦР ֽ, 먦 Ԩ ֬з - 馨 禠 ب. 砤 - ᨦР ֽ֢ 礷 禠 - (Ʒ 礷 㷷) з ب Мֽ - Ҩ , ֢ Ԩ ֬ - ֬Р ҨԷ ; Ш - Ҡ, ᨦ, Ԩ зР 󷤷 - ֽ. - - (Ш) - 笷 (471) | ǠԦ | - - - - ѷ ҠԦ 19921996 ֦ Ʒ շ - - РԦ Ư (Ԭ. Orlando Magic) ֪Ԡ ƨ - , ؠ - Ԩ ֽ ԪԤ - ՠԠֽ ƨֽ ֤. ֨Է з 1989 - ֦, ֦Ҩ ӷԨֽ Ң, Է - գ. ҠԦ ֦ РԦ, ᷦ, ҠԷ - ֦ Ԡ Ԩ 먽-ᠯ, ֨֬ 2010 ֦. - з Ʒ Ʒ, Ơ Ʒ շ, ԪԷ , - , Ш, ӠƬ, 砽 , Ш - Ǡ. 22 ֦ գ ҠԦ 13 ֦Р Ш-֪, 5 - Р ط Ԡ 2 ֦Р Ԡ գ ( 2004 - 2009 ֦). Է Ԡ 2011 ֦, ި ֽ - ҠԦֽ Р з РԦ. - (Ш) - 笷 (1194) | ǠԦ | - - طƷ - - - ШԷ ط: - ESEFUL.jpg - ط Р Ԩ, Ԅ Է - ө (19882001) - ШԷ : - Australia stub.svg - з - 笷 طƷ | ǠԦ | - -֢Է - - Ơ ֢֬ ؠ (Pan troglodytes) . ؠ - , 먽. - Ơ ֢֬ ؠ (Pan troglodytes) . ؠ - , 먽. - - -Ԡ з ? - - 娽 Ʒب - Ѡ ƨҨ - * Ԡ ֦Ơ ֨ ԠԠֽ ֽ Ԡ طԠ ҷ - 㦨РР ֦ ֦ (Ԡ ) ֽ Р련ֽ - ݠᠬ련. - * Halomonas titanicae з頽 㞨 ԷƯ. - * Ơ بЄƠ Р 馠֬ - 뷽 ᬨֽ . - * Ӡ, , Р - Шֽ . - - بԯ - * ֨ ֨ ҷ (Ԡ ) 禠 з 5 - Ш Ԡ. - * Է 㪨 堷 碯 з Ԡ - ƨ ب. - * зƷ -ҷԷ ք ֢馠 ԠԠֽ - Ԩ. - * ՠؠԷ Ҡ㷠 Ԡ 렦 . - * ֢ШҠ , з ֬ ҷ? ֢馠Р - ᨦԨ 夠ҷ ֽ з - Ԩ. - * ǠԠƷ ƨ Ԡ ᠦ Ơ з, - Ԅ ֜ . - - ᨦԷ | ᵷ | | ע禷 - - 21 ޢ - - ԦƷ - * 1097 㠦з ֵ - * 1520 Ԡ ӠР з Ҩ РƷ 淵 - ƨԠҷ - * 1578 Է ֽ ҷ Ԧ֬ Ơ (Ԡ .) - * 1727 Ʒ ⠬Ʒ ֦ط ֬ - Ш Ƿ堨 - * 1805 ᠪƠ Р ՠШԠ - * 1858 ب תԢ ᪨ , Ԡ ֽ - ب Ԩ ƠƠ - * 1895 Է Ԥ Ԡ 栽, ԷԷ 栽ֽ - зƷ - * 1967 㠦 ݨ堬Ԡ ԷƠҷ ֽ ԠҨ - - ֬ Ʒ娽 ֢ 21 ޢ 2011 ֦ Ʒ޵ - - -ҨԠ ᠢ Ԩз - - ᷬР 㨵 Р 巨 ᠢ Ԡ ҷ - ᨦԨ Ʒ 렵 ҷֽ ֬Ҩ - Ԩᬷ-ԯ. - - ҷƠ ֬Ҡ Ԩᬷ-ԯ - Ԩ Ԩᬷ ӷ -1 -2 - Ԧ֬-ب᷽ᷪ ᨬ Ʒ Buran.jpg - ᢷ 1.01 1.02 2.01 - 2.02 2.03 - ؠ --1 (0.01; -; -01) -Ѹ - (0.02; -02) - (0.03) --2 (0.04) - (0.05) - - (0.06) - (0.15) -? (0.08) -4 -5 - Ө Ơ - Ө 㠦 ֨: Ш : - () ؠ: - עԷ -֬ - ү - ҷ 巨 ӨШԢ ӨШԢֽ ԨШ. - ǠԦ | ᵷ | - -֦ ֨ - - wikt: Ʒ - commons: ƷР - Էз Ҩ b: ƷԷ - ԷƷ ֦ - q: ƷԷ - ШƤ m: Ө-Ʒ - Ʒ ֨堵 ƷҨ n: Ʒ - - s: ƷƠ - ᷬԠ wikispecies: Ʒ뷦 - ֬Ʒ 뷦 v: Ʒ - עԷ - - / Embassy ֢Ԡ ᨦ Ԧ - ƷҨ 笷 - Է - http://ru.wikipedia.org/w/index.php?title=%D0%97%D0%B0%D0%B3%D0%BB%D0% - B0%D0%B2%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D - 0%B0&oldid=37831798 - Ǡ娬: - * - -ѷ Ҩ - - * ᨦ / ᨬ - - ҄ - - * РԠ - * ע馨Է - -ᷠ - - - - * Է - * - * - * - - - -ַ - - ____________________ (Submit) ՠ - -ՠ뷬 - - * РԠ Է - * Ơ - * Ơ - * - * Ԡ - * ֢ - -巨 - - * ֢ ֢ ƨ - * ֢ - * - * 鷨 Ʒ - * Է - * Ơ - * Է - -ݨ/ - - * Է - * Ơ Ơ PDF - * ب - -Ҩ - - * Ʒ 㜦 - * Ʒ - * بԷ - * Ԡ Ơ - * Է - -ՠ 笷 Ơ - - * - * Azrbaycanca - * РƠ - * РƠ (ƨ늤) - * ЬƷ - * esky - * Р - * Dansk - * Deutsch - * - * English - * Espaol - * Eesti - * - * Suomi - * Franais - * - * Magyar - * - * Italiano - * - * - * - * - * Lietuvi - * Latvieu - * Nederlands - * Norsk (bokml) - * - * Polski - * Portugus - * Romn - * 䠵 Р - * Ʒ / Srpski - * Svenska - * Trke - * /Tatara - * ᠌Ơ - * Ting Vit - * - - * ШԨ ҨԨԷ ֽ Է: 11:55, 19 ޢ 2011. - * ب з Creative Commons Attribution-ShareAlike; - 妨 ޵ ֬ Է . - ֦֢Ԩ . Է. - Wikipedia ᨬԠ ҠƠ Wikimedia - Foundation, Inc., ԨҨֽ ᬠԷ󠤷. - * Ԡҷ - - * зƠ ԪԤ - * طԷ Ʒب - * Ơ - * ֢Ԡ - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/ru/KOI8-R/wikitop_ru_KOI8-R.txt b/src/tests/testdata/ru/KOI8-R/wikitop_ru_KOI8-R.txt deleted file mode 100644 index 16ab218..0000000 --- a/src/tests/testdata/ru/KOI8-R/wikitop_ru_KOI8-R.txt +++ /dev/null @@ -1,312 +0,0 @@ - # (ru) copyright Atom- - - - - - 19 2011. - . - / - 19 2011. - . - : , - - - , - - , . - 779 821 . - - - - - - - - - - - - - - - - ( ) - - - - - II - - Σ - 3 (16) 1907 II - , - . - - II - , - - . . , , - (-, - -, ) - . II , 20 1907 , - , I . - , - , , - . - - ( ) - , - ţ ; - , , - . - - () - (471) | | - - - - 19921996 - - (. Orlando Magic) - , - - . 1989 - , , - . , , - -, 2010 . ţ - , , , - , , , , - . 22 13 -, 5 - 2 ( 2004 - 2009 ). 2011 , - . - () - (1194) | | - - - - - : - ESEFUL.jpg - , Σ - (19882001) - : - Australia stub.svg - - | | - - - - (Pan troglodytes) ԣ. - , . - (Pan troglodytes) ԣ. - , . - - - ? - - - - * - ( ) - . - * Halomonas titanicae . - * ̣ - . - * , , - . - - - * ( ) 5 - . - * - . - * - ϣ - . - * . - * , ? ݣ - - . - * , - Σ . - - | | | - - 21 - - - * 1097 - * 1520 - - * 1578 ( .) - * 1727 - - * 1805 - * 1858 , - - * 1895 , - - * 1967 - - 21 2011 - - - - - - - -. - ОϞ - - - -1 -2 - - Buran.jpg - 1.01 1.02 2.01 - 2.02 2.03 - --1 (0.01; -; -01) - - (0.02; -02) - (0.03) --2 (0.04) - (0.05) - - (0.06) - (0.15) -? (0.08) -4 -5 - - : : - () : - - - - . - | | - - - - wikt: - commons: - b: - - q: - m: - - n: - - s: - wikispecies: - v: - - - / Embassy - - - http://ru.wikipedia.org/w/index.php?title=%D0%97%D0%B0%D0%B3%D0%BB%D0% - B0%D0%B2%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D - 0%B0&oldid=37831798 - : - * - - - - * / - - ͣ - - * - * - - - - - - * - * - * - * - - - - - - ____________________ (Submit) - - - - * - * - * - * - * - * - - - - * - * - * - * - * - * - * - -/ - - * - * PDF - * - - - - * - * - * - * - * - - - - * - * Azrbaycanca - * - * () - * - * esky - * - * Dansk - * Deutsch - * - * English - * Espaol - * Eesti - * - * Suomi - * Franais - * - * Magyar - * - * Italiano - * - * - * - * - * Lietuvi - * Latvieu - * Nederlands - * Norsk (bokml) - * - * Polski - * Portugus - * Romn - * - * / Srpski - * Svenska - * Trke - * /Tatara - * - * Ting Vit - * - - * : 11:55, 19 2011. - * Creative Commons Attribution-ShareAlike; - . - . . - Wikipedia Wikimedia - Foundation, Inc., . - * - - * - * - * - * - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/ru/UTF-8/wikitop_ru_UTF-8.txt b/src/tests/testdata/ru/UTF-8/wikitop_ru_UTF-8.txt deleted file mode 100644 index afbe7cc..0000000 --- a/src/tests/testdata/ru/UTF-8/wikitop_ru_UTF-8.txt +++ /dev/null @@ -1,312 +0,0 @@ - #Википедия (ru) copyright Википедия — Atom-лента - -Заглавная страница - - Материал из Википедии — свободной энциклопедии - Стабильная версия была проверена 19 сентября 2011. Имеются - непроверенные изменения в шаблонах или файлах. - Текущая версия показать/скрыть подробности - Стабильная версия была проверена 19 сентября 2011. Имеются - непроверенные изменения в шаблонах или файлах. - Перейти к: навигация, поиск - - - Добро пожаловать в Википедию, - - свободную энциклопедию, которую может редактировать каждый. - Сейчас в Википедии 779 821 статья на русском языке. - - География - Искусство - История - Музыка - - Наука - Общество - Персоналии - Религия - - Спорт - Техника - Философия - Все порталы - Создать статью (с помощником) Справка Система рубрикации Пресса о - Википедии - -Избранная статья - - Схема распределения депутатов II Думы по фракциям - - «Третьеиюньский переворот» — широко распространённое название - произошедшего 3 (16) июня 1907 года в России досрочного роспуска II - Государственной Думы, сопровождавшегося изменением избирательной - системы. - - Причиной роспуска II Думы послужила невозможность наладить - конструктивное взаимодействие между правительством, возглавляемым - премьер-министром П. А. Столыпиным, и Думой, значительную часть которой - составляли представители крайних левых партий (социал-демократы, - социалисты-революционеры, народные социалисты) и примыкавшие к ним - трудовики. II Дума, открывшаяся 20 февраля 1907 года, имела не менее - оппозиционное настроение, чем ранее распущенная I Дума. Дума проявила - тенденцию к отклонению всех правительственных законопроектов и бюджета, - а законопроекты, предлагавшиеся Думой, заведомо не могли быть - утверждены Государственным Советом и императором. Сложившаяся ситуация - представляла собой конституционный кризис — Основные государственные - законы (фактически конституция России) позволяли императору в любой - момент распустить Думу, но он был обязан созвать новую Думу и не мог - без её согласия изменить избирательный закон; но при этом следующая - Дума, предположительно, не отличалась бы по оппозиционности от - распущенной. - - (далее…) - Другие избранные статьи (471) | Кандидаты | Просмотр - -Хорошая статья - - Лидер команды в 1992—1996 годы Шакил О’Нил - - «Орландо Мэджик» (англ. Orlando Magic) — профессиональный баскетбольный - клуб, выступающий в Юго-Восточном дивизионе Восточной конференции - Национальной баскетбольной ассоциации. Клуб присоединился к лиге в 1989 - году, одновременно с «Миннесотой Тимбервулвз», в результате расширения - НБА. Команда базируется в городе Орландо, Флорида, и домашние игры - проводит на арене «Эмвей-центра», построенного в 2010 году. В её - составе играли такие игроки, как Шакил О’Нил, Анферни Хардуэй, Грант - Хилл, Рашид Уоллес, Трэйси Макгрэди, Дуайт Ховард, Бен Уоллес и Винс - Картер. За 22 года в НБА команда 13 раз выходила в плей-офф, 5 раз - становилась чемпионом дивизиона и 2 раза выходила в финал НБА (в 2004 и - 2009 годах). По состоянию на 2011 год, клуб является единственной - командой из главных спортивных лиг США в Орландо. - (далее…) - Другие хорошие статьи (1194) | Кандидаты | Просмотр - -Избранные списки и порталы - - - Последний избранный список: - ESEFUL.jpg - Список главных тренеров, выигравших упразднённые клубные турниры - КОНМЕБОЛ (1988—2001) - Последний избранный портал: - Australia stub.svg - Австралия - Другие избранные списки и порталы | Кандидаты | Просмотр - -Изображение дня - - Самка обыкновенного шимпанзе (Pan troglodytes) с детёнышем. Зоопарк - Госсау, Швейцария. - Самка обыкновенного шимпанзе (Pan troglodytes) с детёнышем. Зоопарк - Госсау, Швейцария. - Просмотр - -Знаете ли вы? - - Из новых статей Википедии - Лариса Рикельме - * Активная поддержка своей национальной сборной на чемпионате мира по - футболу сделала одну модель (на фото) самой высокооплачиваемой в - Парагвае. - * Halomonas titanicae в ближайшем будущем съест «Титаник». - * Детская пелёнка стала символом гражданского протеста против - действий аргентинской хунты. - * Мастер, изготовлявший шляпы для фильмов о Гарри Поттере, делал - головные уборы и для королевской семьи. - - Дерево «Гиперион» - * Самое высокое в мире дерево (на фото) удалось вычислить только 5 - лет назад. - * Для создания атмосферы таинственности фильм «Инкуб» целиком снят на - языке эсперанто. - * Австралийский премьер-министр в своё время побеждал в национальной - викторине. - * Нападение марсиан на Эквадор завершилось гибелью шести человек. - * Проблема «что, если бы Бог создал множество миров?» обсуждалась ещё - в Средневековье отцами церкви и служит причиной религиозных - диспутов и поныне. - * Канадский хоккеист был настолько рад выигрышу Кубка Стэнли, что - решил крестить в нём свою дочь. - - Предложения | Архив | Просмотр | Обсудить - -В этот день — 21 октября - - Венденский замок - * 1097 — крестоносцы осадили Антиохию - * 1520 — Фернан Магеллан открыл пролив между Атлантическим и Тихим - океанами - * 1578 — поражение русской армии у стен Венденского замка (на илл.) - * 1727 — русский уполномоченный Савва Рагузинский подписал договор о - торговле с Китаем - * 1805 — Трафальгарская битва оставила Наполеона без флота - * 1858 — премьера оперетты Оффенбаха «Орфей в аду», на которой - впервые был исполнен канкан - * 1895 — вторжение японцев на Тайвань, низвержение Тайваньской - республики - * 1967 — осада Пентагона противниками войны во Вьетнаме - - Логотип «Викиновостей» События 21 октября 2011 года в Викиновостях - Просмотр - -Совместная работа недели - - Приглашаем всех желающих принять участие в работе над статьями о - средневековых русских княжествах и о космической программе - «Энергия-Буран». - п·о·р - Космическая программа «Энергия-Буран» - Компоненты Буран · Энергия · Мир · Квант-1 · Квант-2 · Кристалл · - Андрогинно-периферийный агрегат стыковки Buran.jpg - Орбитальные экземпляры Буран 1.01 · Буран 1.02 · Буран 2.01 · Буран - 2.02 · Буран 2.03 - Тестовые экземпляры и аппараты ОК-МЛ-1 (0.01; ОК-М; БТС-01) · ОК-ГЛИ - (0.02; БТС-02) · ОК-КС (0.03) · ОК-МЛ-2 (0.04) · ОК-ТВА (0.05) · ОК-ТВИ - (0.06) · ОК-МТ (0.15) · ОК-? (0.08) · БОР-4 · БОР-5 - Место запуска Байконур - Места посадок основное: Юбилейный · резервные: Багерово · Восточный - (Хороль) · запасные: прочие - Связанные темы Обрушение крыши монтажно-испытательного корпуса с - «Бураном» - Примите также участие в Мекленбург «Мекленбургской неделе». - Кандидаты | Архив | Просмотр - -Родственные проекты - - wikt: Викисловарь - Словарь и тезаурус commons: Викисклад - Хранилище медиафайлов b: Викиучебник - Учебники и руководства - q: Викицитатник - Коллекция цитат m: Мета-вики - Вики о проектах Викимедиа n: Викиновости - Новости - s: Викитека - Оригинальные тексты wikispecies: Викивиды - Биологические виды v: Викиверситет - Обучение - - Посольство / Embassy — Мобильная версия — Сбор средств — Фонд - Викимедиа — Контакты — Другие языковые разделы - Источник — - «http://ru.wikipedia.org/w/index.php?title=%D0%97%D0%B0%D0%B3%D0%BB%D0% - B0%D0%B2%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D - 0%B0&oldid=37831798» - Категория: - * Порталы - -Личные инструменты - - * Представиться / зарегистрироваться - -Пространства имён - - * Заглавная - * Обсуждение - -Варианты - -Просмотры - - * Чтение - * Текущая версия - * Просмотр - * История - -Действия - -Поиск - - ____________________ (Submit) Найти - -Навигация - - * Заглавная страница - * Рубрикация - * Указатель А — Я - * Избранные статьи - * Случайная статья - * Текущие события - -Участие - - * Сообщить об ошибке - * Портал сообщества - * Форум - * Свежие правки - * Новые страницы - * Справка - * Пожертвования - -Печать/экспорт - - * Создать книгу - * Скачать как PDF - * Версия для печати - -Инструменты - - * Ссылки сюда - * Связанные правки - * Спецстраницы - * Постоянная ссылка - * Цитировать страницу - -На других языках - - * العربية - * Azərbaycanca - * Беларуская - * ‪Беларуская (тарашкевіца)‬ - * Български - * Česky - * Чӑвашла - * Dansk - * Deutsch - * Ελληνικά - * English - * Español - * Eesti - * فارسی - * Suomi - * Français - * עברית - * Magyar - * Հայերեն - * Italiano - * 日本語 - * ქართული - * Қазақша - * 한국어 - * Lietuvių - * Latviešu - * Nederlands - * ‪Norsk (bokmål)‬ - * Ирон - * Polski - * Português - * Română - * Саха тыла - * Српски / Srpski - * Svenska - * Türkçe - * Татарча/Tatarça - * Українська - * Tiếng Việt - * 中文 - - * Последнее изменение этой страницы: 11:55, 19 сентября 2011. - * Текст доступен по лицензии Creative Commons Attribution-ShareAlike; - в отдельных случаях могут действовать дополнительные условия. - Подробнее см. Условия использования. - Wikipedia® — зарегистрированная торговая марка Wikimedia - Foundation, Inc., некоммерческой организации. - * Свяжитесь с нами - - * Политика конфиденциальности - * Описание Википедии - * Отказ от ответственности - * Мобильная версия - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/ru/WINDOWS-1251/wikitop_ru_WINDOWS-1251.txt b/src/tests/testdata/ru/WINDOWS-1251/wikitop_ru_WINDOWS-1251.txt deleted file mode 100644 index c7bbd82..0000000 --- a/src/tests/testdata/ru/WINDOWS-1251/wikitop_ru_WINDOWS-1251.txt +++ /dev/null @@ -1,312 +0,0 @@ - # (ru) copyright Atom- - - - - - 19 2011. - . - / - 19 2011. - . - : , - - - , - - , . - 779 821 . - - - - - - - - - - - - - - - - ( ) - - - - - II - - - 3 (16) 1907 II - , - . - - II - , - - . . , , - (-, - -, ) - . II , 20 1907 , - , I . - , - , , - . - - ( ) - , - ; - , , - . - - () - (471) | | - - - - 19921996 Β - - (. Orlando Magic) - , - - . 1989 - , , - . , , - -, 2010 . - , Β, , - , , , , - . 22 13 -, 5 - 2 ( 2004 - 2009 ). 2011 , - . - () - (1194) | | - - - - - : - ESEFUL.jpg - , - (19882001) - : - Australia stub.svg - - | | - - - - (Pan troglodytes) . - , . - (Pan troglodytes) . - , . - - - ? - - - - * - ( ) - . - * Halomonas titanicae . - * - . - * , , - . - - - * ( ) 5 - . - * - . - * - - . - * . - * , ? - - . - * , - . - - | | | - - 21 - - - * 1097 - * 1520 - - * 1578 ( .) - * 1727 - - * 1805 - * 1858 , - - * 1895 , - - * 1967 - - 21 2011 - - - - - - - -. - - - - -1 -2 - - Buran.jpg - 1.01 1.02 2.01 - 2.02 2.03 - --1 (0.01; -; -01) - - (0.02; -02) - (0.03) --2 (0.04) - (0.05) - - (0.06) - (0.15) -? (0.08) -4 -5 - - : : - () : - - - - . - | | - - - - wikt: - commons: - b: - - q: - m: - - n: - - s: - wikispecies: - v: - - - / Embassy - - - http://ru.wikipedia.org/w/index.php?title=%D0%97%D0%B0%D0%B3%D0%BB%D0% - B0%D0%B2%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D - 0%B0&oldid=37831798 - : - * - - - - * / - - - - * - * - - - - - - * - * - * - * - - - - - - ____________________ (Submit) - - - - * - * - * - * - * - * - - - - * - * - * - * - * - * - * - -/ - - * - * PDF - * - - - - * - * - * - * - * - - - - * - * Azrbaycanca - * - * () - * - * esky - * - * Dansk - * Deutsch - * - * English - * Espaol - * Eesti - * - * Suomi - * Franais - * - * Magyar - * - * Italiano - * - * - * - * - * Lietuvi - * Latvieu - * Nederlands - * Norsk (bokml) - * - * Polski - * Portugus - * Romn - * - * / Srpski - * Svenska - * Trke - * /Tatara - * - * Ting Vit - * - - * : 11:55, 19 2011. - * Creative Commons Attribution-ShareAlike; - . - . . - Wikipedia Wikimedia - Foundation, Inc., . - * - - * - * - * - * - - * Wikimedia Foundation - * Powered by MediaWiki diff --git a/src/tests/testdata/ru/X-MAC-CYRILLIC/wikitop_ru_MACCYRILLIC.txt b/src/tests/testdata/ru/X-MAC-CYRILLIC/wikitop_ru_MACCYRILLIC.txt deleted file mode 100644 index 03738d0..0000000 --- a/src/tests/testdata/ru/X-MAC-CYRILLIC/wikitop_ru_MACCYRILLIC.txt +++ /dev/null @@ -1,157 +0,0 @@ - # (ru) copyright Atom- - - - - - 19 2011. - . - / - 19 2011. - . - : , - - - , - - , . - 779 821 . - - - - - - - - - - - - - - - - ( ) - - - - - II - - ǒ - 3 (16) 1907 II - , - . - - II - , - - . . , , - (-, - -, ) - . II , 20 1907 , - , I . - , - , , - . - - ( ) - , - ; - , , - . - - () - (471) | | - - - - 19921996 Ս - - ǎ (. Orlando Magic) - , - - . 1989 - , nj , - . , , - ǝ-, 2010 . - , Ս, , - , , , , - . 22 13 -, 5 - 2 ( 2004 - 2009 ). 2011 , - . - () - (1194) | | - - - - - : - ESEFUL.jpg - , - (19882001) - : - Australia stub.svg - - | | - - - - (Pan troglodytes) . - , . - (Pan troglodytes) . - , . - - - ? - - - - * - ( ) - . - * Halomonas titanicae ǒ. - * - . - * , , - . - - ǃ - * ( ) 5 - . - * Lj - . - * - - . - * . - * , ? - - . - * , - . - - | | | - - 21 - - - * 1097 - * 1520 - - * 1578 ( .) - * 1727 - - * 1805 - * 1858 ǎ , - - * 1895 , - - * 1967 - - ǂ 21 2011 - - - - - - - ǝ-. - \ No newline at end of file diff --git a/src/tests/testdata/ru/ibm855.txt b/src/tests/testdata/ru/ibm855.txt new file mode 100644 index 0000000..6e19fc6 --- /dev/null +++ b/src/tests/testdata/ru/ibm855.txt @@ -0,0 +1,5 @@ +CP855 -- ƷзƠ ֦ Է MS-DOS ֦֢ بᠤ . М Ԡ ISO 8859-5. + +ը֬ Р ᢷ, ӠƨԷ Ь᷷, 㷷 Ԩ Р. + +ݨ Ԡ ֽ֦ 堢з ؠ ASCII. Р ֦ ҷ ֢Ԡ Ԡ ֦ Է֦. diff --git a/src/tests/testdata/ru/ibm866.txt b/src/tests/testdata/ru/ibm866.txt new file mode 100644 index 0000000..0dc4775 --- /dev/null +++ b/src/tests/testdata/ru/ibm866.txt @@ -0,0 +1,11 @@ +<<ୠ⨢ ஢>> (<<ୠ⨢ ஢ >>) -- ᭮ +CP437 ࠭, ᯥ᪨ ய᪨ ᨬ ன + ਫ, ⠢ ᥢ᪨ ᨬ ஭묨. +⥫쭮, ணࠬ, ᯮ ࠡ ⥪⮢ +, ⠪ ᯥ稢 ᯮ짮 ᨬ ਫ. + +᪨ ⢮ ਠ⮢ ୠ⨢ ஢, +ࠧ ⮫쪮 0xF0 -- 0xFF (240--255). ᯮ짮 +ᥢ 䨪-ᠬ, ᠢ訥 ᨬ 묨 +ணࠬ⠬, ࠭ १ । +業. diff --git a/src/tests/testdata/ru/iso-8859-5.txt b/src/tests/testdata/ru/iso-8859-5.txt new file mode 100644 index 0000000..6246c8d --- /dev/null +++ b/src/tests/testdata/ru/iso-8859-5.txt @@ -0,0 +1,3 @@ +ISO 8859-5 +8- ISO-8859 . +ISO 8859-5 " " ( , ). diff --git a/src/tests/testdata/ru/koi8-r.txt b/src/tests/testdata/ru/koi8-r.txt new file mode 100644 index 0000000..1972c27 --- /dev/null +++ b/src/tests/testdata/ru/koi8-r.txt @@ -0,0 +1 @@ +-8 ( , 8 ), KOI8 - , ASCII. . Unix- , 2010 , , . diff --git a/src/tests/testdata/ru/maccyrillic.txt b/src/tests/testdata/ru/maccyrillic.txt new file mode 100644 index 0000000..ad849a3 --- /dev/null +++ b/src/tests/testdata/ru/maccyrillic.txt @@ -0,0 +1,9 @@ + MacCyrillic nj. + + + ; , + , ( ), +, . + + () +ASCII. . diff --git a/src/tests/testdata/ru/windows-1251.txt b/src/tests/testdata/ru/windows-1251.txt new file mode 100644 index 0000000..c76f0be --- /dev/null +++ b/src/tests/testdata/ru/windows-1251.txt @@ -0,0 +1,4 @@ +Windows-1251 + , 8- Microsoft Windows. . , Windows 19901991 . , Microsoft. ( , ). + +Windows-1251 8- ( CP866, KOI8-R ISO 8859-5) , ( ); : , , , . diff --git a/src/tests/testdata/se/UTF-8/wikitop_se_UTF-8.txt b/src/tests/testdata/se/UTF-8/wikitop_se_UTF-8.txt deleted file mode 100644 index 7870d09..0000000 --- a/src/tests/testdata/se/UTF-8/wikitop_se_UTF-8.txt +++ /dev/null @@ -1,158 +0,0 @@ -Váldosiidu - - Wikipedias - Njuike: navigašuvdna, oza - Bures boahtin Wikipediai, almmolaš diehtosátnegirjái. - Davvisámegielat Wikipedias lea dál oktiibuot 3,756 artihkkala. - Geahča buot kategoriijaid | Historjá | Kultuvra | Luondu | Lottit | - Riikkat | Gielat | Sápmi | Šattut - -Dieđut Wikipedias - - Wikipedia lea máŋggagielat prošeakta man ulbmilin lea ráhkadit almmolaš - diehtosátnegirjji masa gii beare sáhttá čállit artihkkaliid. Maid don - sáhtát juogo dievasmahttit artihkkaliid dahje čálligoahtit áibbas ođđa - artihkkala muhtun fáttás. Sámegielat Wikipedias leat dál 3,756 - artihkkala. - - Wikipedia váldosiiddus leat eambbo dieđut - -Mo don sáhtát veahkehit - - Wikipedia dárbbaša du veahki! Rukses liŋka mearkkaša dan, ahte fáttás - ii leat vel álggahuvvon artihkal. - Don sáhtát veahkehit omd. dainna ahte čálát artihkkala. Dás leat muhtun - siidosávaldagat: - Susan Aglukark | Aráljávri | Biibbal | Bižus | Davisnuorri | - Davviálbmogat | Davvi polárgierdu | Giemajávrri dálongielda | - Jiekŋamearra | Juksáhkká | Aune Kuuva | Arthur Miller | Poava | Ságat | - Suopmelaš-permelaš gielat ‎ | Suopmelaš-volgalaš gielat ‎ | Waren - Sárdne - - Lasi artihkalsávaldagat - -Ovttasbargoartihkal - - Kalaallit Nunaata leavga - - Ovttasbargoartihkkala ulbmilin lea ovttas čállit dahje ođasmahttit - muhtun artihkkala. - - Ovttasbargoartihkal lea Kalaallit Nunaat‎. - - Nuvola filesystems www.png - -[ođasmahte] Áigeguovdil - - Ellen Johnson-Sirleaf detail 071024-D-9880W-027.jpg - Liberia presideanta Ellen Johnson-Sirleaf jagis 2007 - * Bhutana gonagas Jigme Khesar Namgyel Wangchuck ja Jetsun Pema - náitaleigga. (13.10.2011) - * Ellen Johnson-Sirleaf (govas), Leymah Gbowee ja Tawakkul Karman - vuite Nobel ráfibálkkašumi. (7.10.2011) - * Ruoŧŧelaš Tomas Tranströmer vuittii Nobel girjjálašvuođa - bálkkašumi. (6.10.2011) - * Suoma Sámedikki válggat 2011 álge.(5.9.2011) - * Nuortasápmelaš girječálli Kati-Claudia Fofonoff lea - jápman.(12.6.2011) - - Loga lasi ođđasiid - Nuvola apps date.png - -[ođasmahte] golggotmánu 21. beaivi - - Template:Golggotmánu 21. - - Geahča eará beivviid - - Nuvola apps filetypes.png - -[ođasmahte] Dihtetgo, ahte... - - * Ruoŧa ruvdnaprinseassa Victoria isida prinsa Daniela - riegádanbeaivi, čakčamánu 15. b. 1973, lea seamma beaivi go prinsa - vuohppasássa Carl XVI Gustaf šattai Ruoŧa gonagas? - * Qataras eai leat bissovaš jogat eaige jávrrit. - * Jagis 1867 Amerihká ovttastuvvan stáhtat oste Alaska USD 7,2 - miljovnna Ruoššas? - * Jus juogát jahkelogu 4:iin ja olles lohku báhcá, de lea dan jagi - gárgádusjahki, omd 1964:4=491. Jus jahkelogus báhcá rihkku, de ii - leat gárgádusjahkevuođđu, omd 1963:4=490,75. - - Lasi unna dieđut - - Logo Wikipedia eará gielain - -Wikipedia eará gielain - - Sámeguovllu riikkaid gielain - - Norway Girjedárogillii (Bokmål) • Ođđadárogillii (Nynorsk) • - Russia Ruoššagillii (Русский) • Sweden Ruoŧagillii (Svenska) • - Finland Suomagillii (Suomi) - - Badjel 100 000 artihkkala - Eaŋgalsgillii (English) • Dánskkagillii (Dansk) • Duiskkagillii - (Deutsch) • Fránskkagillii (Français) • Polskkagillii (Polski) • - Japánagillii (日本語) • Itáliagillii (Italiano) • Hollánddagillii - (Nederlands) • Portugálagillii(Português) • Spánskagillii - (Español) • Romániagillii (Română)• Ukrainagillii (Українська) • - Ungárgillii (Magyar) • Čeahkagillii (Česká) • Slovákiagillii - (Slovenčina) • Katalánagillii (Català) - - Badjel 10 000 artihkkala - Bulgáriagillii (Български) • Indonesiagillii (Bahasa Indonesia) - • Kroátiagillii (Hrvatski) • Ido • Serbiagillii (Српски) • - Slovenagillii (Slovenščina) • Esttegillii (Eesti) • Albánagillii - (Shqip) - - - Vižžojuvvon čujuhusas - -Persovnnalaš bargoneavvu - - * Daga ođđa geavaheaddjidovddaldaga dahje čálligoađe sisa - - - * Artihkal - * Ságastallan - -Variants - -Pláđemat - - -Oza - -Navigašuvdna - - * Váldosiidu - * Gáffestohpu - * Vuođđoartihkkalat - * Áigeguovdil - * Varas rievdadusat - * Summal siidu - * Veahkki - * Skeaŋkkat - -Gielladikšun - - * Sámi giellatekno - * Analysere ja disambiguere - * Generere paradigmaid - * Generere sániid - * Generere lohkosániid - * Oza báikenamaid - * Jorgalanreaiddut - * Sátnegirji - -Neavvobumbá - - * Siiddut mat čujuhit deikke - * Dán siiddu varas rievdadusat - * Erenomáš siiddut - * Prentenveršuvdna - * Bissovaš liŋka - * Cite this page - -Eará gielaide diff --git a/src/tests/testdata/se/WINDOWS-1252/wikitop_se_WINDOWS-1252.txt b/src/tests/testdata/se/WINDOWS-1252/wikitop_se_WINDOWS-1252.txt deleted file mode 100644 index 3506824..0000000 --- a/src/tests/testdata/se/WINDOWS-1252/wikitop_se_WINDOWS-1252.txt +++ /dev/null @@ -1,16 +0,0 @@ -Under r 1977 deltog fransmannen Thierry Sabine i ett rally mellan Abidjan i Elfenbenskusten och Nice i Frankrike. Han krde dock vilse mitt ute i den libyska knen p sin motorcykel, men blev rddad och kunde tervnda till Frankrike. Sabine blev mycket fascinerad av landskapet i den afrikanska knen och ville dela med sig av upplevelsen till s mnga som mjligt. Han skapade d ett rally, kallat Paris-Dakar-rallyt, vilket skulle g frn Frankrikes huvudstad Paris, ned genom Algeriet och Agadez i Niger, fr att slutligen g i ml i Dakar, Senegals huvudstad.[1] Det frsta rallyt krdes under rsskiftet 1978/1979 och det blev fransk seger i bde bil- och motorcykelklassen. Strckan har sedan varierat genom ren, men fokus har koncentrerats till Afrikaregionen, fram till r 2007. Oroligheter i lnder rallyt korsat har vid ett flertal tillfllen varit ett stort problem fr arrangrerna. Bland annat har strckor stllts in eller bytts ut efter hot om terroristattacker, kidnappningar, plundringar samt vid ett tillflle ocks minflt. - -Den 4 januari 2008 meddelades det att rets tvling, som skulle hllits 5-20 januari 2008, stlldes in p grund av terroristhot i Mauretanien. Ngra dagar senare kom ett besked om att det fanns planer p att anordna tvlingen i Ungern och Rumnien i maj 2008. Dakarrallyt 2008 stlldes dock in och i stllet startade Centraleuropeiska rallyt och Dakarserien. Dakarrallyt hade hllits utan avbrott varje r sedan 1979, men det hade regelbundet lagts fram krav p att stlla in det p grund av skerhetsrisk och den fara snabbgende fordon utgr fr lokalbefolkningarna, men 2008 var frsta gngen kraven gick igenom.[2] - -Under perioden 3-18 januari 2009 krdes den 31:a upplagan av Dakarrallyt och den frsta som inte gick genom Afrika. Rallyt hade istllet flyttats till Sydamerika, med lnderna Argentina och Chile.[3]. Mycket tyder p att tvlingen kommer att fortstta kras i Sydamerika, d dessa tvlingar fungerat nstintill felfritt i frhllande till ren i Afrika. - -Utver knda rallyfrare, som exempelvis Colin McRae, Carlos Sainz och Ari Vatanen, har Dakarrallyt ven uppmrksammats efter att flertalet andra knda personer deltagit i rallyt, bland annat strtloppskarna Luc Alphand och Bernhard Russi, affrsmannen Mark Thatcher, Albert II av Monaco och Caroline av Hannover.[1] -[redigera] Kritik -Marc Coma under Dakarrallyt 2009. - -Rallyt har blivit knt fr att vara en av vrlden farligaste tvlingar, vilket den ftt mycket kritik fr. Sedan start har ett sextiotal personer dtt i samband med rallyt, 25 av dessa har varit frare som dtt i singelolyckor, men det har ven hnt att frare tappat kontrollen ver sitt fordon och krt ut bland publiken. ven tv helikopterkrascher har frekommit och i en av dessa omkom rallyts grundare, Thierry Sabine. - -Arrangrer har tagit hrt p kritiken och har infrt flera tgrder fr att hlla ned hastigheterna fr de tvlande. I vissa byar har bland annat dolda hastighetskontroller infrts dr strafftid lagt till de tvlande som krt fr snabbt.[4] Annat r att frsvra navigeringen s att deltagarna mste koncentrera sig mer p att navigera rtt istllet fr att vinna p hastighet. Fordonen har ven utrustats med varningslampor som lyser nr de deltagande kommer nra varandra fr att undvika kollisioner. - -Sedan 2012 rs upplaga har frarna i motorcykelklassen frbjudits kra med motorer strre n 450 kubik.[5] Detta infrande har varit kontroversiellt, eftersom flera deltagare sjlva menar att det inte r till deras frdel. -[redigera] diff --git a/src/tests/testdata/sk/ibm852.txt b/src/tests/testdata/sk/ibm852.txt new file mode 100644 index 0000000..725a54f --- /dev/null +++ b/src/tests/testdata/sk/ibm852.txt @@ -0,0 +1,3 @@ +Jupiter je piata planta v porad od Slnka, najvia a najhmotnejia planta +naej slnenej sstavy. Je pomenovan po rmskom bohovi Jupiterovi. Symbolom +planty je tylizovan znzornenie Jupiterovho boskho blesku. diff --git a/src/tests/testdata/sk/iso-8859-2.txt b/src/tests/testdata/sk/iso-8859-2.txt new file mode 100644 index 0000000..ee3ab14 --- /dev/null +++ b/src/tests/testdata/sk/iso-8859-2.txt @@ -0,0 +1,3 @@ +Jupiter je piata planta v porad od Slnka, najvia a najhmotnejia planta +naej slnenej sstavy. Je pomenovan po rmskom bohovi Jupiterovi. Symbolom +planty je tylizovan znzornenie Jupiterovho boskho blesku. diff --git a/src/tests/testdata/sk/maccentraleurope.txt b/src/tests/testdata/sk/maccentraleurope.txt new file mode 100644 index 0000000..cddbba4 --- /dev/null +++ b/src/tests/testdata/sk/maccentraleurope.txt @@ -0,0 +1,3 @@ +Jupiter je piata planta v porad od Slnka, najvia a najhmotnejia planta +naej slnenej sstavy. Je pomenovan po rmskom bohovi Jupiterovi. Symbolom +planty je tylizovan znzornenie Jupiterovho boskho blesku. diff --git a/src/tests/testdata/sk/utf-8.txt b/src/tests/testdata/sk/utf-8.txt new file mode 100644 index 0000000..eba4382 --- /dev/null +++ b/src/tests/testdata/sk/utf-8.txt @@ -0,0 +1,3 @@ +Jupiter je piata planéta v poradí od Slnka, najväčšia a najhmotnejšia planéta +našej slnečnej sústavy. Je pomenovaný po rímskom bohovi Jupiterovi. Symbolom +planéty je štylizované znázornenie Jupiterovho božského blesku. diff --git a/src/tests/testdata/sk/windows-1250.txt b/src/tests/testdata/sk/windows-1250.txt new file mode 100644 index 0000000..a60d048 --- /dev/null +++ b/src/tests/testdata/sk/windows-1250.txt @@ -0,0 +1,3 @@ +Jupiter je piata planta v porad od Slnka, najvia a najhmotnejia planta +naej slnenej sstavy. Je pomenovan po rmskom bohovi Jupiterovi. Symbolom +planty je tylizovan znzornenie Jupiterovho boskho blesku. diff --git a/src/tests/testdata/sl/ibm852.txt b/src/tests/testdata/sl/ibm852.txt new file mode 100644 index 0000000..5fa60a4 --- /dev/null +++ b/src/tests/testdata/sl/ibm852.txt @@ -0,0 +1,9 @@ +Naseljvi plant je planet ali naravni satelit (redkeje tudi asteroid[1]), ki je +zmoen razviti in ohranjati ivljenje. + +Ker je obstoj nezemeljskega ivljenja trenutno negotov, je raziskovanje +naseljivih planetov v glavnem ekstrapolacija razmer na Zemlji in znailnosti +Sonca in celotnega Osonja, ki govorijo v prid razvitju ivljenja. e posebej so +pomembni faktorji, ki so ohranili zapletene, mnogoceline organizme in ne le +preprosta, enocelina iva bitja, mikroorganizme. Raziskovanje in teorija v tej +smeri je del planetologije in razvijajoe astrobiologije. diff --git a/src/tests/testdata/sl/iso-8859-16.txt b/src/tests/testdata/sl/iso-8859-16.txt new file mode 100644 index 0000000..80d0b26 --- /dev/null +++ b/src/tests/testdata/sl/iso-8859-16.txt @@ -0,0 +1,9 @@ +Naseljvi plant je planet ali naravni satelit (redkeje tudi asteroid[1]), ki je +zmoen razviti in ohranjati ivljenje. + +Ker je obstoj nezemeljskega ivljenja trenutno negotov, je raziskovanje +naseljivih planetov v glavnem ekstrapolacija razmer na Zemlji in znailnosti +Sonca in celotnega Osonja, ki govorijo v prid razvitju ivljenja. e posebej so +pomembni faktorji, ki so ohranili zapletene, mnogoceline organizme in ne le +preprosta, enocelina iva bitja, mikroorganizme. Raziskovanje in teorija v tej +smeri je del planetologije in razvijajoe astrobiologije. diff --git a/src/tests/testdata/sl/iso-8859-2.txt b/src/tests/testdata/sl/iso-8859-2.txt new file mode 100644 index 0000000..7af252e --- /dev/null +++ b/src/tests/testdata/sl/iso-8859-2.txt @@ -0,0 +1,9 @@ +Naseljvi plant je planet ali naravni satelit (redkeje tudi asteroid[1]), ki je +zmoen razviti in ohranjati ivljenje. + +Ker je obstoj nezemeljskega ivljenja trenutno negotov, je raziskovanje +naseljivih planetov v glavnem ekstrapolacija razmer na Zemlji in znailnosti +Sonca in celotnega Osonja, ki govorijo v prid razvitju ivljenja. e posebej so +pomembni faktorji, ki so ohranili zapletene, mnogoceline organizme in ne le +preprosta, enocelina iva bitja, mikroorganizme. Raziskovanje in teorija v tej +smeri je del planetologije in razvijajoe astrobiologije. diff --git a/src/tests/testdata/sl/maccentraleurope.txt b/src/tests/testdata/sl/maccentraleurope.txt new file mode 100644 index 0000000..4e84135 --- /dev/null +++ b/src/tests/testdata/sl/maccentraleurope.txt @@ -0,0 +1,9 @@ +Naseljvi plant je planet ali naravni satelit (redkeje tudi asteroid[1]), ki je +zmoen razviti in ohranjati ivljenje. + +Ker je obstoj nezemeljskega ivljenja trenutno negotov, je raziskovanje +naseljivih planetov v glavnem ekstrapolacija razmer na Zemlji in znailnosti +Sonca in celotnega Osonja, ki govorijo v prid razvitju ivljenja. e posebej so +pomembni faktorji, ki so ohranili zapletene, mnogoceline organizme in ne le +preprosta, enocelina iva bitja, mikroorganizme. Raziskovanje in teorija v tej +smeri je del planetologije in razvijajoe astrobiologije. diff --git a/src/tests/testdata/sl/utf-8.txt b/src/tests/testdata/sl/utf-8.txt new file mode 100644 index 0000000..11d013b --- /dev/null +++ b/src/tests/testdata/sl/utf-8.txt @@ -0,0 +1,9 @@ +Naseljívi planét je planet ali naravni satelit (redkeje tudi asteroid[1]), ki je +zmožen razviti in ohranjati življenje. + +Ker je obstoj nezemeljskega življenja trenutno negotov, je raziskovanje +naseljivih planetov v glavnem ekstrapolacija razmer na Zemlji in značilnosti +Sonca in celotnega Osončja, ki govorijo v prid razvitju življenja. Še posebej so +pomembni faktorji, ki so ohranili zapletene, mnogocelične organizme in ne le +preprosta, enocelična živa bitja, mikroorganizme. Raziskovanje in teorija v tej +smeri je del planetologije in razvijajoče astrobiologije. diff --git a/src/tests/testdata/sl/windows-1250.txt b/src/tests/testdata/sl/windows-1250.txt new file mode 100644 index 0000000..512309b --- /dev/null +++ b/src/tests/testdata/sl/windows-1250.txt @@ -0,0 +1,9 @@ +Naseljvi plant je planet ali naravni satelit (redkeje tudi asteroid[1]), ki je +zmoen razviti in ohranjati ivljenje. + +Ker je obstoj nezemeljskega ivljenja trenutno negotov, je raziskovanje +naseljivih planetov v glavnem ekstrapolacija razmer na Zemlji in znailnosti +Sonca in celotnega Osonja, ki govorijo v prid razvitju ivljenja. e posebej so +pomembni faktorji, ki so ohranili zapletene, mnogoceline organizme in ne le +preprosta, enocelina iva bitja, mikroorganizme. Raziskovanje in teorija v tej +smeri je del planetologije in razvijajoe astrobiologije. diff --git a/src/tests/testdata/sv/iso-8859-1.txt b/src/tests/testdata/sv/iso-8859-1.txt new file mode 100644 index 0000000..fcf070c --- /dev/null +++ b/src/tests/testdata/sv/iso-8859-1.txt @@ -0,0 +1,10 @@ +Mlle r en ttort p Kullahalvn i Brunnby socken i Hgans kommun, Skne ln. + +Samhllet var frn brjan ett fiskelge, men kom att spela en stor roll i den +framvxande turismen i Sverige i slutet av 1800-talet. Till detta bidrog - och +bidrar - Mlles naturskna lge invid resunds norra utlopp, med Kullaberg som +bakgrund. Gemensamhetsbad fr mn och kvinnor introducerades i Ransvik i brjan +av 1900-talet. Storhetstiden som turistort intrffade strax fre frsta +vrldskriget, men ven under mellankrigstiden var turiststrmmarna stora. +Fortfarande r Mlle en populr turistort med en tredubbling av invnarantalet +under sommarmnaderna. diff --git a/src/tests/testdata/sv/utf-8.txt b/src/tests/testdata/sv/utf-8.txt new file mode 100644 index 0000000..d66be04 --- /dev/null +++ b/src/tests/testdata/sv/utf-8.txt @@ -0,0 +1,10 @@ +Mölle är en tätort på Kullahalvön i Brunnby socken i Höganäs kommun, Skåne län. + +Samhället var från början ett fiskeläge, men kom att spela en stor roll i den +framväxande turismen i Sverige i slutet av 1800-talet. Till detta bidrog – och +bidrar – Mölles natursköna läge invid Öresunds norra utlopp, med Kullaberg som +bakgrund. Gemensamhetsbad för män och kvinnor introducerades i Ransvik i början +av 1900-talet. Storhetstiden som turistort inträffade strax före första +världskriget, men även under mellankrigstiden var turistströmmarna stora. +Fortfarande är Mölle en populär turistort med en tredubbling av invånarantalet +under sommarmånaderna. diff --git a/src/tests/testdata/sv/windows-1252.txt b/src/tests/testdata/sv/windows-1252.txt new file mode 100644 index 0000000..94f15c6 --- /dev/null +++ b/src/tests/testdata/sv/windows-1252.txt @@ -0,0 +1,10 @@ +Mlle r en ttort p Kullahalvn i Brunnby socken i Hgans kommun, Skne ln. + +Samhllet var frn brjan ett fiskelge, men kom att spela en stor roll i den +framvxande turismen i Sverige i slutet av 1800-talet. Till detta bidrog och +bidrar Mlles naturskna lge invid resunds norra utlopp, med Kullaberg som +bakgrund. Gemensamhetsbad fr mn och kvinnor introducerades i Ransvik i brjan +av 1900-talet. Storhetstiden som turistort intrffade strax fre frsta +vrldskriget, men ven under mellankrigstiden var turiststrmmarna stora. +Fortfarande r Mlle en populr turistort med en tredubbling av invnarantalet +under sommarmnaderna. diff --git a/src/tests/testdata/th/TIS-620/utffool_th_TIS-620.txt b/src/tests/testdata/th/TIS-620/utffool_th_TIS-620.txt deleted file mode 100644 index 7d47d2e..0000000 --- a/src/tests/testdata/th/TIS-620/utffool_th_TIS-620.txt +++ /dev/null @@ -1,283 +0,0 @@ -˹ѡ - - ҡԡԾ ʹء - 价: º͡ҧ, - 24 Ҥ 2554, 16:30 ê - - Թյ͹ѺԡԾ - ҹءշءö - ԡԾ 69,781 - - ______________________ ¡ - - : §ѡ Ǵ | ˹ҷ - - Ѵ͹ - Ѵ͹ - Ҿʴдк - ͢Ҵͧǵѵǹԧ - ҧҧѵǹԧ - - к Сͺ´ǧҷԵѵ - ⤨ͺǧҷԵͧҡçǧ 8 - ǧѺǧѹ÷鹾 166 ǧ 5 - ǧѺǧѹ÷鹾 4 ǧ ѺѵآҴ - աѺҹ ֧ ѵᶺ ҧ - 紴 нҧ - - ·Ǩҹҧ ͧк - ѺҡǧҷԵ͡Ҵѧ 㹨ӹǹ 4 ǧ - ᶺ 좹Ҵ˭ͺ͡ӹǹ 4 ǧ - ᶺ觻Сͺѵط繨Ѵ繹 - 鹨ҡᶺ͡ࢵᶺҹШ ͺࢵ; - (ࢵᴹɮշ鹡ѧŧͧҡҧǧ) - о仨ҡ鹤ҹͧ - - ʾҷ͡ҡǧҷԵ () - 价к - ҧçҴ˭ҧ¡ѹ - 觢͡仨ҡ㨡ҧͧᶺҹШ - - ͡ 8 ǧк - §ӴѺҡǧҷԵش͡ մѧ Ǿظ ء š - ѧ Ǿʺ ù д๻ٹ (ҹ...) - - Ѵá͹˹ҹ: á´Ժ ҾشҪԹ Ԫ - - 纶 Ѵ - - ... - - ͧʹ㨨ҡشͧԡԾ : - - Ǻúҧǧͧù - * ... Ǻâͧù ѡ 27 ǧ (ҧǹҾ) - ժ͵Ф㹼ŧҹûоѹͧ ડ - 硫ѹ ⻻ - * ... .. 2553 ҡҹҹҪҵਹͧѺ 11.7 - ҹ ǺԹ 170,000 - * ... - ŹԸԡШçê͡ҧ㹡ͧѭҵ - ҧ˹ѡͧä - * ... .. 2490 줴ӹ֡ 385 - ªԵҡਹ - Ѻ繹ѡӹӤáªԵѧ͡ਹ - * ... - öʵҢ˹֡ǡѺŤ¨ҡѭѡɳ - ͧЪʶҹóкԺҾǴ੾ - -ͧҹ ҧ ʹͺ - - ͧҡ - ͧҡ - Ѵҿ - * ¡Թᴹʡ շ СȨصԡ͹Ǵظ - * ʹյ Ѵҿ (Ҿ) ªԵ - * Դ˵ءóط¤˭ Դ 61 - ѧѴǻ - * ҪԸɡҧ 稾ҪҸԺ - 稾ҪԹٯҹ Ѵ žٹҤ ٯҹ - * ʵտ ͺ ͵駺ѷͻ - ªԵä秵Ѻ͹ 56 - * ᴹ પѹ ʵҨŨҡԷ൵ - ѺҧҢջШӻ 2554 - - ˵ءóѨغѹ - - ѹʹյ - ѹʹյ - 24 Ҥ: ѹ˻ЪҪҵ; ѹ͡Ҫ (.. 2507) - - ѵҺѹʹѭֹ - * .. 1803 (.. 1260) êҷ ͧҷ Ƚ - վԸҧġ ¾ 9 觽 - * .. 2191 (.. 1648) ʹѭֹ - ʹѭҩѺͧʹѭѹҾʵ Ѻŧ - (Ҿ) ռصʧԺ㹷ջû - СûѵŹ - ѺͧҸóѰŹ˾ѹѰ - Ѱ͡Ҫ - * .. 2472 (.. 1929) ÿصિŴ - ˹ÿصŷÿصԷ·ش - šҷաúѹ֡ ١Ѵ駢 - * .. 2474 (.. 1931) оҹ ͪԧѹ - 觻ѨغѹѺ˹оҹաèҨ˹蹷ش˹ - 觢ͧš ҧù졡Ѻ Ѱ - ӾԸҧġ - * .. 2488 (.. 1945) ѵ˻ЪҪҵ - ٭ͧͧ˻ЪҪҵ ռźѧѺ - ѧҡѵҺѹͧҸóѰչ Ҿµ - ҪҳҨѡ Ѱԡ ЪҵԷŧǹ˭ - - ѹ͹˹ҹ: 23 Ҥ 22 Ҥ 21 Ҥ - - ˵͹Ҥ - - - ҹء - ҹء - - * Ǵ:ҵ ҵ - * Ǵ:Ż Ż - - * Ǵ:Էʵ Էʵ - * Ǵ:ʵ ʵ - - * Ǵ:෤ ෤ - * Ǵ: - - * Ǵ:ѵʵ ѵʵ - * Ǵ:ѧ ѧ - - * Ǵ: - * ԡԾ:ʶҹ ʶҹ - -º͡ҧ - - * һЪҤ дҹ ç - ѾҡСԨ觤ͺԡԾҧҧҧ - * ŢҪԡԾ ʧǡѺҹԡԾ - * ب-Ѫ ʧ·价سҡ - * 䫵 С Ѿവ - ТŢǡѺԡԾŹԸԡ - * Ҫ ѺԻǡѺԡԾ - ֧§ҹѭ෤Ԥʹ͹º - * Local Embassy For Wikipedia-related discussion in languages other - than Thai. - -çþͧ - - ԡԾ´ԹŹԸԡ ͧǧҼš - 繼Թաç - - ԡԫ - - ԡԫ - ͡õ鹩Ѻ - - ͹ - - ͹ - ٹ - - ԡԾҹء - - ԡԾҹء - ҹءöԸҹ - - ԡԵ - - ԡԵ - Ф - - ԡԤӤ - - ԡԤӤ - ǺӾٴ - - ԡ - - ԡ -ٹҹҹçԡ - - ԡʻժ - - ԡʻժ - ú͹ءԸҹ - - - - ͡ҡ ԡԾѧա 269 ҷš - ԡԾ¢Ҵ˭ҡªʹҹҧ - * 500,000 : Deutsch (ѹ) English (ѧ) - Espaol (໹) Franais () Italiano (Ե) - () Nederlands (ѵ) Polski (Ź) Portugus - (õ) () - - * 150,000 : Catal (ҵѹ) etina () Suomi - (ԹŹ) Magyar (ѧ) Norsk (bokml) () Romn - () Svenska (ഹ) Trke (á) - (ù) (չ) - - * 100,000 : (Ѻ) Dansk (ഹ) - Esperanto (ѹ) (պ) Bahasa Indonesia - (Թⴹ) () Lietuvi (Է) Bahasa Melayu - () Slovenina (ѡ) Slovenina () / - Srpski () Ting Vit (´) Volapk (һء) - Winaray (-) - - * 50,000 : () (ա) - Eesti () Euskara (ʡ) () Gallego - () (Թ) Hrvatski () Kryol ayisyen - (ε) () Norsk (nynorsk) (չʡ) - Armneashce (¹) Simple English (ѧҧ) - - ֧Ũҡ - -ͧǹ - - * ͡Թ / ҧѭռ - -໫ - - * - * Ի - -觷ᵡҧ - - - - * - * - * ѵ - -áз - -׺ - - ____________________ (Submit) ׺ - -º͡ҧ - - * ˹ѡ - * ˵ءóѨغѹ - * Ӷ - * Ѵ - * سҾ - * - -ǹ - - * һЪҤ - * Ѻاش - * ¹ҹ - * ԴԡԾ - * ԨҤԡԾ - * Ը - -/͡ - - * ҧ˹ѧ - * ǹŴ㹪 PDF - * ˹Ѻ - -ͧ - - * ˹ҷԧ - * Ѻا§ - * ѻŴ - * ˹Ҿ - * ԧ - * ҧԧ - - - * ˹ҹشѹ 2 ѹ¹ 2554 18:54 . - * ͹حҵѭ͹حҵͷտ͹ - Ẻʴ-͹حҵẺǡѹ; ͹ҨСͺ - ô֡͹䢡ҹ - Wikipedia ͧ¡äҨ¹ͧŹԸԡ - * Դ - - * º¤ǹ - * ǡѺԡԾ - * ͻʸѺԴͺ diff --git a/src/tests/testdata/th/TIS-620/wikitop_th_TIS-620.txt b/src/tests/testdata/th/TIS-620/wikitop_th_TIS-620.txt deleted file mode 100644 index 52d6ae9..0000000 --- a/src/tests/testdata/th/TIS-620/wikitop_th_TIS-620.txt +++ /dev/null @@ -1,283 +0,0 @@ -˹ѡ - - ҡԡԾ ҹء - 价: º͡ҧ, - 24 Ҥ 2554, 16:30 ê - - Թյ͹ѺԡԾ - ҹءշءö - ԡԾ 69,781 - - ______________________ ¡ - - : §ѡ Ǵ | ˹ҷ - - Ѵ͹ - Ѵ͹ - Ҿʴдк - ͢Ҵͧǵѵǹԧ - ҧҧѵǹԧ - - к Сͺ´ǧҷԵѵ - ⤨ͺǧҷԵͧҡçǧ 8 - ǧѺǧѹ÷鹾 166 ǧ 5 - ǧѺǧѹ÷鹾 4 ǧ ѺѵآҴ - աѺҹ ֧ ѵᶺ ҧ - 紴 нҧ - - ·Ǩҹҧ ͧк - ѺҡǧҷԵ͡Ҵѧ 㹨ӹǹ 4 ǧ - ᶺ 좹Ҵ˭ͺ͡ӹǹ 4 ǧ - ᶺ觻Сͺѵط繨Ѵ繹 - 鹨ҡᶺ͡ࢵᶺҹШ ͺࢵ; - (ࢵᴹɮշ鹡ѧŧͧҡҧǧ) - о仨ҡ鹤ҹͧ - - ʾҷ͡ҡǧҷԵ () - 价к - ҧçҴ˭ҧ¡ѹ - 觢͡仨ҡ㨡ҧͧᶺҹШ - - ͡ 8 ǧк - §ӴѺҡǧҷԵش͡ մѧ Ǿظ ء š - ѧ Ǿʺ ù д๻ٹ (ҹ...) - - Ѵá͹˹ҹ: á´Ժ ҾشҪԹ Ԫ - - 纶 Ѵ - - ... - - ͧʹ㨨ҡشͧԡԾ : - - Ǻúҧǧͧù - * ... Ǻâͧù ѡ 27 ǧ (ҧǹҾ) - ժ͵Ф㹼ŧҹûоѹͧ ડ - 硫ѹ ⻻ - * ... .. 2553 ҡҹҹҪҵਹͧѺ 11.7 - ҹ ǺԹ 170,000 - * ... - ŹԸԡШçê͡ҧ㹡ͧѭҵ - ҧ˹ѡͧä - * ... .. 2490 줴ӹ֡ 385 - ªԵҡਹ - Ѻ繹ѡӹӤáªԵѧ͡ਹ - * ... - öʵҢ˹֡ǡѺŤ¨ҡѭѡɳ - ͧЪʶҹóкԺҾǴ੾ - -ͧҹ ҧ ʹͺ - - ͧҡ - ͧҡ - Ѵҿ - * ¡Թᴹʡ շ СȨصԡ͹Ǵظ - * ʹյ Ѵҿ (Ҿ) ªԵ - * Դ˵ءóط¤˭ Դ 61 - ѧѴǻ - * ҪԸɡҧ 稾ҪҸԺ - 稾ҪԹٯҹ Ѵ žٹҤ ٯҹ - * ʵտ ͺ ͵駺ѷͻ - ªԵä秵Ѻ͹ 56 - * ᴹ પѹ ʵҨŨҡԷ൵ - ѺҧҢջШӻ 2554 - - ˵ءóѨغѹ - - ѹʹյ - ѹʹյ - 24 Ҥ: ѹ˻ЪҪҵ; ѹ͡Ҫ (.. 2507) - - ѵҺѹʹѭֹ - * .. 1803 (.. 1260) êҷ ͧҷ Ƚ - վԸҧġ ¾ 9 觽 - * .. 2191 (.. 1648) ʹѭֹ - ʹѭҩѺͧʹѭѹҾʵ Ѻŧ - (Ҿ) ռصʧԺ㹷ջû - СûѵŹ - ѺͧҸóѰŹ˾ѹѰ - Ѱ͡Ҫ - * .. 2472 (.. 1929) ÿصિŴ - ˹ÿصŷÿصԷ·ش - šҷաúѹ֡ ١Ѵ駢 - * .. 2474 (.. 1931) оҹ ͪԧѹ - 觻ѨغѹѺ˹оҹաèҨ˹蹷ش˹ - 觢ͧš ҧù졡Ѻ Ѱ - ӾԸҧġ - * .. 2488 (.. 1945) ѵ˻ЪҪҵ - ٭ͧͧ˻ЪҪҵ ռźѧѺ - ѧҡѵҺѹͧҸóѰչ Ҿµ - ҪҳҨѡ Ѱԡ ЪҵԷŧǹ˭ - - ѹ͹˹ҹ: 23 Ҥ 22 Ҥ 21 Ҥ - - ˵͹Ҥ - - - ҹء - ҹء - - * Ǵ:ҵ ҵ - * Ǵ:Ż Ż - - * Ǵ:Էʵ Էʵ - * Ǵ:ʵ ʵ - - * Ǵ:෤ ෤ - * Ǵ: - - * Ǵ:ѵʵ ѵʵ - * Ǵ:ѧ ѧ - - * Ǵ: - * ԡԾ:ʶҹ ʶҹ - -º͡ҧ - - * һЪҤ дҹ ç - ѾҡСԨ觤ͺԡԾҧҧҧ - * ŢҪԡԾ ʧǡѺҹԡԾ - * ب-Ѫ ʧ·价سҡ - * 䫵 С Ѿവ - ТŢǡѺԡԾŹԸԡ - * Ҫ ѺԻǡѺԡԾ - ֧§ҹѭ෤Ԥʹ͹º - * Local Embassy For Wikipedia-related discussion in languages other - than Thai. - -çþͧ - - ԡԾ´ԹŹԸԡ ͧǧҼš - 繼Թաç - - ԡԫ - - ԡԫ - ͡õ鹩Ѻ - - ͹ - - ͹ - ٹ - - ԡԾҹء - - ԡԾҹء - ҹءöԸҹ - - ԡԵ - - ԡԵ - Ф - - ԡԤӤ - - ԡԤӤ - ǺӾٴ - - ԡ - - ԡ -ٹҹҹçԡ - - ԡʻժ - - ԡʻժ - ú͹ءԸҹ - - - - ͡ҡ ԡԾѧա 269 ҷš - ԡԾ¢Ҵ˭ҡªʹҹҧ - * 500,000 : Deutsch (ѹ) English (ѧ) - Espaol (໹) Franais () Italiano (Ե) - () Nederlands (ѵ) Polski (Ź) Portugus - (õ) () - - * 150,000 : Catal (ҵѹ) etina () Suomi - (ԹŹ) Magyar (ѧ) Norsk (bokml) () Romn - () Svenska (ഹ) Trke (á) - (ù) (չ) - - * 100,000 : (Ѻ) Dansk (ഹ) - Esperanto (ѹ) (պ) Bahasa Indonesia - (Թⴹ) () Lietuvi (Է) Bahasa Melayu - () Slovenina (ѡ) Slovenina () / - Srpski () Ting Vit (´) Volapk (һء) - Winaray (-) - - * 50,000 : () (ա) - Eesti () Euskara (ʡ) () Gallego - () (Թ) Hrvatski () Kryol ayisyen - (ε) () Norsk (nynorsk) (չʡ) - Armneashce (¹) Simple English (ѧҧ) - - ֧Ũҡ - -ͧǹ - - * ͡Թ / ҧѭռ - -໫ - - * - * Ի - -觷ᵡҧ - - - - * - * - * ѵ - -áз - -׺ - - ____________________ (Submit) ׺ - -º͡ҧ - - * ˹ѡ - * ˵ءóѨغѹ - * Ӷ - * Ѵ - * سҾ - * - -ǹ - - * һЪҤ - * Ѻاش - * ¹ҹ - * ԴԡԾ - * ԨҤԡԾ - * Ը - -/͡ - - * ҧ˹ѧ - * ǹŴ㹪 PDF - * ˹Ѻ - -ͧ - - * ˹ҷԧ - * Ѻا§ - * ѻŴ - * ˹Ҿ - * ԧ - * ҧԧ - - - * ˹ҹشѹ 2 ѹ¹ 2554 18:54 . - * ͹حҵѭ͹حҵͷտ͹ - Ẻʴ-͹حҵẺǡѹ; ͹ҨСͺ - ô֡͹䢡ҹ - Wikipedia ͧ¡äҨ¹ͧŹԸԡ - * Դ - - * º¤ǹ - * ǡѺԡԾ - * ͻʸѺԴͺ diff --git a/src/tests/testdata/th/UTF-8/wikitop_th_UTF-8.txt b/src/tests/testdata/th/UTF-8/wikitop_th_UTF-8.txt deleted file mode 100644 index cc03f5f..0000000 --- a/src/tests/testdata/th/UTF-8/wikitop_th_UTF-8.txt +++ /dev/null @@ -1,283 +0,0 @@ -หน้าหลัก - - จากวิกิพีเดีย สารานุกรมเสรี - ไปที่: ป้ายบอกทาง, ค้นหา - 24 ตุลาคม 2554, 16:30 – รีเฟรช - - ยินดีต้อนรับสู่วิกิพีเดีย - สารานุกรมเสรีที่ทุกคนสามารถแก้ไขได้ - วิกิพีเดียภาษาไทยมี 69,781 บทความ - - ______________________ เรียกดู ค้นหา - - ค้นหา: เรียงตามตัวอักษร • หมวดหมู่ | ดูหน้าทั้งหมด - - บทความคัดสรรเดือนนี้ - บทความคัดสรรเดือนนี้ - ภาพแสดงดาวเคราะห์และดาวเคราะห์แคระในระบบสุริยะ - โดยย่อขนาดของดาวตามอัตราส่วนจริง - แต่ระยะห่างระหว่างดาวไม่ใช่อัตราส่วนจริง - - ระบบสุริยะ ประกอบด้วยดวงอาทิตย์และวัตถุอื่นๆ - ที่โคจรรอบดวงอาทิตย์เนื่องจากแรงโน้มถ่วง ได้แก่ ดาวเคราะห์ 8 - ดวงกับดวงจันทร์บริวารที่ค้นพบแล้ว 166 ดวง ดาวเคราะห์แคระ 5 - ดวงกับดวงจันทร์บริวารที่ค้นพบแล้ว 4 ดวง กับวัตถุขนาดเล็กอื่นๆ - อีกนับล้านชิ้น ซึ่งรวมถึง ดาวเคราะห์น้อย วัตถุในแถบไคเปอร์ ดาวหาง - สะเก็ดดาว และฝุ่นระหว่างดาวเคราะห์ - - โดยทั่วไปแล้วจะแบ่งย่านต่างๆ ของระบบสุริยะ - นับจากดวงอาทิตย์ออกมาดังนี้คือ ดาวเคราะห์ชั้นในจำนวน 4 ดวง - แถบดาวเคราะห์น้อย ดาวเคราะห์ขนาดใหญ่รอบนอกจำนวน 4 ดวง - และแถบไคเปอร์ซึ่งประกอบด้วยวัตถุที่เย็นจัดเป็นน้ำแข็ง - พ้นจากแถบไคเปอร์ออกไปเป็นเขตแถบจานกระจาย ขอบเขตเฮลิโอพอส - (เขตแดนตามทฤษฎีที่ซึ่งลมสุริยะสิ้นกำลังลงเนื่องจากมวลสารระหว่างดวงดาว) - และพ้นไปจากนั้นคือย่านของเมฆออร์ต - - กระแสพลาสมาที่ไหลออกจากดวงอาทิตย์ (หรือลมสุริยะ) - จะแผ่ตัวไปทั่วระบบสุริยะ - สร้างโพรงขนาดใหญ่ขึ้นในสสารระหว่างดาวเรียกกันว่า เฮลิโอสเฟียร์ - ซึ่งขยายออกไปจากใจกลางของแถบจานกระจาย - - ดาวเคราะห์ชั้นเอกทั้ง 8 ดวงในระบบสุริยะ - เรียงลำดับจากใกล้ดวงอาทิตย์ที่สุดออกไป มีดังนี้คือ ดาวพุธ ดาวศุกร์ โลก - ดาวอังคาร ดาวพฤหัสบดี ดาวเสาร์ ดาวยูเรนัส และดาวเนปจูน (อ่านต่อ...) - - บทความคัดสรรก่อนหน้านี้: อสุรกายดงดิบ – ภาพชุดพระราชินีมารี เดอ เมดิชิ - – ประเทศเปรู - ที่เก็บถาวร – บทความคัดสรรอื่น ๆ - รู้ไหมว่า - รู้ไหมว่า... - - เรื่องน่าสนใจจากบทความล่าสุดของวิกิพีเดีย : - - ดาวบริวารบางดวงของดาวยูเรนัส - * ... ดาวบริวารของดาวยูเรนัส ที่รู้จักแล้ว 27 ดวง (บางส่วนในภาพ) - ทั้งหมดมีชื่อตามตัวละครในผลงานการประพันธ์ของวิลเลียม เชกสเปียร์ - และอเล็กซันเดอร์ โปป - * ... เมื่อ พ.ศ. 2553 ท่าอากาศยานนานาชาติเจนีวารองรับผู้โดยสาร 11.7 - ล้านคน และเที่ยวบินกว่า 170,000 เที่ยว - * ... - มูลนิธิกระจกเงามีโครงการช่วยเป็นสื่อกลางในการยื่นเรื่องขอสัญชาติไทย - แก่ชาวเขา และสร้างความตระหนักเรื่องการค้ามนุษย์ - * ... เมื่อ พ.ศ. 2490 มอริส ฟาร์คดำน้ำลึก 385 เมตร - และเสียชีวิตด้วยอาการเมาไนโตรเจน - นับเป็นนักดำน้ำคนแรกที่เสียชีวิตขณะใช้ถังออกซิเจน - * ... - อรรถศาสตร์สาขาหนึ่งศึกษาเกี่ยวกับการแปลความหมายจากป้ายและสัญลักษณ์ท - ี่เหล่าองค์กรและชุมชนใช้ในสถานการณ์และบริบทหรือสภาพแวดล้อมเฉพาะ - -เรื่องที่ผ่านมา – สร้างบทความใหม่ – เสนอบทความ - - เรื่องจากข่าว - เรื่องจากข่าว - มูอัมมาร์ กัดดาฟี - * กลุ่มแบ่งแยกดินแดนบาสก์ อีทีเอ ประกาศจะยุติการเคลื่อนไหวด้วยอาวุธ - * อดีตผู้นำลิเบีย มูอัมมาร์ กัดดาฟี (ในภาพ) เสียชีวิตแล้ว - * เกิดเหตุการณ์อุทกภัยครั้งใหญ่ ก่อให้เกิดความเสียหาย 61 - จังหวัดทั่วประเทศไทย - * พระราชพิธีอภิเษกสมรสระหว่าง สมเด็จพระราชาธิบดี - และสมเด็จพระราชินีแห่งภูฏาน จัดขึ้น ณ มณฑลพูนาคา ประเทศภูฏาน - * สตีฟ จอบส์ ผู้ร่วมก่อตั้งบริษัทแอปเปิล - เสียชีวิตด้วยโรคมะเร็งตับอ่อน รวมอายุได้ 56 ปี - * แดน เชชท์มัน ศาสตราจารย์ชาวอิสราเอลจากมหาวิทยาลัยไอโอวาสเตต - ได้รับรางวัลโนเบลสาขาเคมีประจำปี 2554 - - เหตุการณ์ปัจจุบัน - - วันนี้ในอดีต - วันนี้ในอดีต - 24 ตุลาคม: วันสหประชาชาติ; วันเอกราชในแซมเบีย (พ.ศ. 2507) - - การให้สัตยาบันในสนธิสัญญามึนสเตอร์ - * พ.ศ. 1803 (ค.ศ. 1260) – มหาวิหารชาทร์ ในเมืองชาทร์ ประเทศฝรั่งเศส - มีพิธีวางศิลาฤกษ์ โดยพระเจ้าหลุยส์ที่ 9 แห่งฝรั่งเศส - * พ.ศ. 2191 (ค.ศ. 1648) – สนธิสัญญามึนสเตอร์ - สนธิสัญญาฉบับที่สองในสนธิสัญญาสันติภาพเวสต์ฟาเลีย ได้รับการลงนาม - (ในภาพ) ซึ่งมีผลยุติสงครามสามสิบปีในทวีปยุโรป - และการปฏิวัติเนเธอร์แลนด์ - และมีเนื้อหาให้การรับรองสาธารณรัฐเจ็ดสหเนเธอร์แลนด์และสหพันธรัฐสวิส - เป็นรัฐเอกราช - * พ.ศ. 2472 (ค.ศ. 1929) – สโมสรฟุตบอลเชฟฟิลด์ - หนึ่งในสโมสรฟุตบอลที่มิใช่สโมสรฟุตบอลในมหาวิทยาลัยที่เก่าแก่ที่สุดใ - นโลกเท่าที่เคยมีการบันทึก ถูกจัดตั้งขึ้น - * พ.ศ. 2474 (ค.ศ. 1931) – สะพานจอร์จ วอชิงตัน - ซึ่งปัจจุบันนับว่าเป็นหนึ่งในสะพานที่มีการจราจรหนาแน่นที่สุดแห่งหนึ - ่งของโลก เชื่อมระหว่างนครนิวยอร์กกับฟอร์ตลี รัฐนิวเจอร์ซี - ทำพิธีวางศิลาฤกษ์ - * พ.ศ. 2488 (ค.ศ. 1945) – กฎบัตรสหประชาชาติ - ธรรมนูญขององค์การสหประชาชาติ มีผลบังคับใช้ - ภายหลังจากการให้สัตยาบันของสาธารณรัฐจีน ฝรั่งเศส สหภาพโซเวียต - สหราชอาณาจักร สหรัฐอเมริกา และชาติที่ร่วมลงนามส่วนใหญ่ - - วันก่อนหน้านี้: 23 ตุลาคม – 22 ตุลาคม – 21 ตุลาคม - - จดหมายเหตุเดือนตุลาคม - - - สารานุกรม - สารานุกรม - - * หมวดหมู่:ธรรมชาติ ธรรมชาติ - * หมวดหมู่:ศิลปะ ศิลปะ - - * หมวดหมู่:วิทยาศาสตร์ วิทยาศาสตร์ - * หมวดหมู่:ภูมิศาสตร์ ภูมิศาสตร์ - - * หมวดหมู่:เทคโนโลยี เทคโนโลยี - * หมวดหมู่:ความเชื่อ ความเชื่อ - - * หมวดหมู่:ประวัติศาสตร์ ประวัติศาสตร์ - * หมวดหมู่:สังคม สังคม - - * หมวดหมู่:ประเทศไทย ประเทศไทย - * วิกิพีเดีย:สถานีย่อย สถานีย่อย - -ป้ายบอกทาง - - * ศาลาประชาคม – กระดานข่าว โครงการ - ทรัพยากรและกิจกรรมซึ่งครอบคลุมวิกิพีเดียอย่างกว้างขวาง - * เลขาชาววิกิพีเดีย – ถามข้อสงสัยเกี่ยวกับการใช้งานวิกิพีเดีย - * ปุจฉา-วิสัชนา – ถามข้อสงสัยทั่วไปที่คุณอยากรู้ - * ข่าวไซต์ – ประกาศ อัพเดต - บทความและข้อมูลข่าวเกี่ยวกับวิกิพีเดียและมูลนิธิวิกิมีเดีย - * ศาลาชุมชน – สำหรับอภิปรายเกี่ยวกับวิกิพีเดีย - รวมถึงรายงานปัญหาเทคนิคและเสนอนโยบาย - * Local Embassy – For Wikipedia-related discussion in languages other - than Thai. - -โครงการพี่น้อง - - วิกิพีเดียดำเนินการโดยมูลนิธิวิกิมีเดีย องค์กรไม่แสวงหาผลกำไร - ซึ่งเป็นผู้ดำเนินการอีกหลายโครงการ ได้แก่ - - วิกิซอร์ซ - - วิกิซอร์ซ - เอกสารต้นฉบับเสรี - - คอมมอนส์ - - คอมมอนส์ - ศูนย์รวมสื่อเสรี - - วิกิพจนานุกรม - - วิกิพจนานุกรม - พจนานุกรมและอรรถาภิธาน - - วิกิตำรา - - วิกิตำรา - ตำราและคู่มือเสรี - - วิกิคำคม - - วิกิคำคม - แหล่งรวบรวมคำพูด - - เมต้าวิกิ - - เมต้าวิกิ -ศูนย์ประสานงานโครงการวิกิมีเดีย - - วิกิสปีชีส์ - - วิกิสปีชีส์ - สารบบอนุกรมวิธาน - -ภาษาอื่น - - นอกจากภาษาไทยแล้ว วิกิพีเดียยังมีรุ่นภาษาอื่นอีก 269 ภาษาทั่วโลก - โดยวิกิพีเดียขนาดใหญ่ปรากฏในรายชื่อด้านล่าง - * 500,000 บทความขึ้นไป: Deutsch (เยอรมัน) • English (อังกฤษ) • - Español (สเปน) • Français (ฝรั่งเศส) • Italiano (อิตาลี) • 日本語 - (ญี่ปุ่น) • Nederlands (ดัตช์) • Polski (โปแลนด์) • Português - (โปรตุเกส) • Русский (รัสเซีย) - - * 150,000 บทความขึ้นไป: Català (คาตาลัน) • Čeština (เช็ก) • Suomi - (ฟินแลนด์) • Magyar (ฮังการี) • Norsk (bokmål) (นอร์เวย์) • Română - (โรมาเนีย) • Svenska (สวีเดน) • Türkçe (ตุรกี) • Українська - (ยูเครน) • 中文 (จีน) - - * 100,000 บทความขึ้นไป: العربية (อาหรับ) • Dansk (เดนมาร์ก) • - Esperanto (เอสเปรันโต) • עברית (ฮีบรู) • Bahasa Indonesia - (อินโดนีเซีย) • 한국어 (เกาหลี) • Lietuvių (ลิทัวเนีย) • Bahasa Melayu - (มลายู) • Slovenčina (สโลวัก) • Slovenščina (สโลวีเนีย) • Српски / - Srpski (เซอร์เบีย) • Tiếng Việt (เวียดนาม) • Volapük (โวลาปุก) • - Winaray (วาราย-วาราย) - - * 50,000 บทความขึ้นไป: Български (บัลแกเรีย) • Ελληνικά (กรีก) • - Eesti (เอสโตเนีย) • Euskara (บาสก์) • فارسی (เปอร์เซีย) • Gallego - (กาลิเซีย) • हिन्दी (ฮินดี) • Hrvatski (โครเอเชีย) • Krèyol ayisyen - (เฮติ) • नेपाल भाषा (เนวารี) • Norsk (nynorsk)‬ (นีนอสก์) • - Armãneashce (อะโรมาเนียน) • Simple English (อังกฤษอย่างง่าย) - - ดึงข้อมูลจาก - -เครื่องมือส่วนตัว - - * ล็อกอิน / สร้างบัญชีผู้ใช้ - -เนมสเปซ - - * บทความ - * อภิปราย - -สิ่งที่แตกต่าง - -ดู - - * เนื้อหา - * ดูโค้ด - * ประวัติ - -การกระทำ - -สืบค้น - - ____________________ (Submit) สืบค้น - -ป้ายบอกทาง - - * หน้าหลัก - * เหตุการณ์ปัจจุบัน - * ถามคำถาม - * บทความคัดสรร - * บทความคุณภาพ - * สุ่มบทความ - -มีส่วนร่วม - - * ศาลาประชาคม - * ปรับปรุงล่าสุด - * เรียนรู้การใช้งาน - * ติดต่อวิกิพีเดีย - * บริจาคให้วิกิพีเดีย - * วิธีใช้ - -พิมพ์/ส่งออก - - * สร้างหนังสือ - * ดาวน์โหลดในชื่อ PDF - * หน้าสำหรับพิมพ์ - -เครื่องมือ - - * หน้าที่ลิงก์มา - * ปรับปรุงที่เกี่ยวโยง - * อัปโหลด - * หน้าพิเศษ - * ลิงก์ถาวร - * อ้างอิงบทความนี้ - -ภาษาอื่น - * หน้านี้แก้ไขล่าสุดเมื่อวันที่ 2 กันยายน 2554 เวลา 18:54 น. - * อนุญาตให้เผยแพร่ภายใต้สัญญาอนุญาตครีเอทีฟคอมมอนส์ - แบบแสดงที่มา-อนุญาตแบบเดียวกัน; เงื่อนไขอื่นอาจใช้ประกอบด้วย - โปรดศึกษาเงื่อนไขการใช้งาน - Wikipedia® เป็นเครื่องหมายการค้าจดทะเบียนของมูลนิธิวิกิมีเดีย - * ติดต่อเรา - - * นโยบายความเป็นส่วนตัว - * เกี่ยวกับวิกิพีเดีย - * ข้อปฏิเสธความรับผิดชอบ diff --git a/src/tests/testdata/th/iso-8859-11.txt b/src/tests/testdata/th/iso-8859-11.txt new file mode 100644 index 0000000..14deb7c --- /dev/null +++ b/src/tests/testdata/th/iso-8859-11.txt @@ -0,0 +1,5 @@ +TIS-620 + +ҵðҹԵѳصˡ 620-2533, ͡.620-2533, ͷѡѹ TIS-620 繪شѡҵðҹصˡͧ ժ Ѻѡ·Ѻ + + TIS-620 ´ ISO-8859-11 ҡ ᵡҧѹ§ ISO-8859-11 ˹ A0 "äẺѴ" (no-break space) ǹ TIS-620 ʧǹ˹ A0 ˹ diff --git a/src/tests/testdata/th/tis-620.txt b/src/tests/testdata/th/tis-620.txt new file mode 100644 index 0000000..0439613 --- /dev/null +++ b/src/tests/testdata/th/tis-620.txt @@ -0,0 +1,5 @@ +TIS-620 + +ҵðҹԵѳصˡ 620-2533, ͡.620-2533, ͷѡѹ TIS-620 繪شѡҵðҹصˡͧ ժ Ѻѡ·Ѻ + + TIS-620 ´ ISO-8859-11 ҡ ᵡҧѹ§ ISO-8859-11 ˹ A0 "äẺѴ" (no-break space) ǹ TIS-620 ʧǹ˹ A0 ˹ diff --git a/src/tests/testdata/th/utf-8.txt b/src/tests/testdata/th/utf-8.txt new file mode 100644 index 0000000..73a9c1a --- /dev/null +++ b/src/tests/testdata/th/utf-8.txt @@ -0,0 +1 @@ +ยูนิโคด (อังกฤษ: Unicode) คือมาตรฐานอุตสาหกรรมที่ช่วยให้คอมพิวเตอร์แสดงผลและจัดการข้อความธรรมดาที่ใช้ในระบบการเขียนของภาษาส่วนใหญ่ในโลกได้อย่างสอดคล้องกัน ยูนิโคดประกอบด้วยรายการอักขระที่แสดงผลได้มากกว่า 100,000 ตัว พัฒนาต่อยอดมาจากมาตรฐานชุดอักขระสากล (Universal Character Set: UCS) และมีการตีพิมพ์ลงในหนังสือ The Unicode Standard เป็นแผนผังรหัสเพื่อใช้เป็นรายการอ้างอิง นอกจากนั้นยังมีการอธิบายวิธีการที่ใช้เข้ารหัสและการนำเสนอมาตรฐานของการเข้ารหัสอักขระอีกจำนวนหนึ่ง การเรียงลำดับอักษร กฎเกณฑ์ของการรวมและการแยกอักขระ รวมไปถึงลำดับการแสดงผลของอักขระสองทิศทาง (เช่นอักษรอาหรับหรืออักษรฮีบรูที่เขียนจากขวาไปซ้าย) diff --git a/src/tests/testdata/tr/ISO-8859-9/wikitop_tr_ISO-8859-9.txt b/src/tests/testdata/tr/ISO-8859-9/wikitop_tr_ISO-8859-9.txt deleted file mode 100644 index c862343..0000000 --- a/src/tests/testdata/tr/ISO-8859-9/wikitop_tr_ISO-8859-9.txt +++ /dev/null @@ -1,33 +0,0 @@ -stanbul, Trkiye'nin en kalabalk, iktisadi ve kltrel adan en nemli -ehri.[2][3][4] ktisadi byklk adan dnyada 34., nfus asndan -belediye snrlar gz nne alnarak yaplan sralamaya gre Avrupa'da -birinci srada gelir.[5][6] - - -stanbul Trkiye'nin kuzeybatsnda, Marmara kys ve Boazii boyunca, -Hali'i de evreleyecek ekilde kurulmutur.[7] stanbul ktalararas bir -ehir olup, Avrupa'daki blmne Avrupa Yakas veya Rumeli Yakas, -Asya'daki blmne ise Anadolu Yakas denir. Tarihte ilk olarak taraf -Marmara Denizi, Boazii ve Hali'in sard bir yarm ada zerinde kurulan -stanbul'un batdaki snrn stanbul Surlar oluturmaktayd. Gelime ve -byme srecinde surlarn her seferinde daha batya ilerletilerek ina -edilmesiyle 4 defa geniletilen ehrin [8] 39 ilesi vardr.[9] Snrlar -ierisinde ise bykehir belediyesi ile birlikte toplam 40 belediye -bulunmaktadr. - - -Dnyann en eski ehirlerinden biri olan stanbul, M.S. 330 - 395 yllar -arasnda Roma mparatorluu, 395 - 1204 ile 1261 - 1453 yllar arasnda -Dou Roma mparatorluu, 1204 - 1261 arasnda Latin mparatorluu ve son -olarak 1453 - 1922 yllar arasnda Osmanl mparatorluu'na bakentlik -yapt.[10] Ayrca, hilafetin Osmanl mparatorluu'na getii 1517'den, -kaldrld 1924'e kadar, stanbul slamiyet'in de merkezi oldu.[11] - -1453 ylnda fetihten sonra, kent Osmanl mparatorluu'nun drdnc -bakenti ilan edilidi ve Kostantiniyye Osmanl mparatorluu tarafndan -kentin resmi ad olarak kullanld ve 1923 ylnda Osmanl -mparatorluunun kne kadar, ou zaman bu ad kullanmda -kald. rnein Osmanl mparatorluu ve mahkemeleri, Kostantiniyye'de -yaymlanan resmi belgelerin kaynan belirtmek iin, "be-Makam- -Dar's-Saltanat- Kostantiniyyet'l-Mahrust'l-Mahmiyye" gibi balklar -kullanlrd.[17] diff --git a/src/tests/testdata/tr/UTF-8/wikitop_tr_UTF-8.txt b/src/tests/testdata/tr/UTF-8/wikitop_tr_UTF-8.txt deleted file mode 100644 index 6c92ad3..0000000 --- a/src/tests/testdata/tr/UTF-8/wikitop_tr_UTF-8.txt +++ /dev/null @@ -1,33 +0,0 @@ -İstanbul, Türkiye'nin en kalabalık, iktisadi ve kültürel açıdan en önemli -şehri.[2][3][4] İktisadi büyüklük açıdan dünyada 34., nüfus açısından -belediye sınırları göz önüne alınarak yapılan sıralamaya göre Avrupa'da -birinci sırada gelir.[5][6] - - -İstanbul Türkiye'nin kuzeybatısında, Marmara kıyısı ve Boğaziçi boyunca, -Haliç'i de çevreleyecek şekilde kurulmuştur.[7] İstanbul kıtalararası bir -şehir olup, Avrupa'daki bölümüne Avrupa Yakası veya Rumeli Yakası, -Asya'daki bölümüne ise Anadolu Yakası denir. Tarihte ilk olarak üç tarafı -Marmara Denizi, Boğaziçi ve Haliç'in sardığı bir yarım ada üzerinde kurulan -İstanbul'un batıdaki sınırını İstanbul Surları oluşturmaktaydı. Gelişme ve -büyüme sürecinde surların her seferinde daha batıya ilerletilerek inşa -edilmesiyle 4 defa genişletilen şehrin [8] 39 ilçesi vardır.[9] Sınırları -içerisinde ise büyükşehir belediyesi ile birlikte toplam 40 belediye -bulunmaktadır. - - -Dünyanın en eski şehirlerinden biri olan İstanbul, M.S. 330 - 395 yılları -arasında Roma İmparatorluğu, 395 - 1204 ile 1261 - 1453 yılları arasında -Doğu Roma İmparatorluğu, 1204 - 1261 arasında Latin İmparatorluğu ve son -olarak 1453 - 1922 yılları arasında Osmanlı İmparatorluğu'na başkentlik -yaptı.[10] Ayrıca, hilafetin Osmanlı İmparatorluğu'na geçtiği 1517'den, -kaldırıldığı 1924'e kadar, İstanbul İslamiyet'in de merkezi oldu.[11] - -1453 yılında fetihten sonra, kent Osmanlı İmparatorluğu'nun dördüncü -başkenti ilan edilidi ve Kostantiniyye Osmanlı İmparatorluğu tarafından -kentin resmi adı olarak kullanıldı ve 1923 yılında Osmanlı -İmparatorluğu’nun çöküşüne kadar, çoğu zaman bu ad kullanımda -kaldı. Örneğin Osmanlı İmparatorluğu ve mahkemeleri, Kostantiniyye'de -yayımlanan resmi belgelerin kaynağını belirtmek için, "be-Makam-ı -Darü's-Saltanat-ı Kostantiniyyetü'l-Mahrusâtü'l-Mahmiyye" gibi başlıklar -kullanılırdı.[17] diff --git a/src/tests/testdata/tr/iso-8859-3.txt b/src/tests/testdata/tr/iso-8859-3.txt new file mode 100644 index 0000000..0cb6dc0 --- /dev/null +++ b/src/tests/testdata/tr/iso-8859-3.txt @@ -0,0 +1,13 @@ +Trke, Trk dili ya da Trkiye Trkesi, batda Balkanlar'dan balayp douda +Hazar Denizi sahasna kadar konuulan Altay dillerinden biridir. Ya, en eski +hesaplara gre 8500 olan Trke, bugn yaayan Dnya dilleri arasnda en eski +yazl belgelere sahip olan dildir. Bu belgeler, ivi yazl Smerce +tabletlerdeki alnt kelimelerdir.[12] Trk yaz dilleri iinde Ouz sahas yaz +dillerinden Osmanl Trkesinin devamn oluturur. Bata Trkiye olmak zere +eski Osmanl mparatorluu corafyasnda konuulan Trke, dnyada en fazla +konuulan 5. dildir. Trke sondan eklemeli bir dildir.[13] Bundan tr +kullanlan herhangi bir eylem zerinden istenildii kadar szck +tretilebilir.[14] Trkiye Trkesi bu ynnden dolay dier Trk dilleriyle +ortak ya da ayrk bulunan onlarca eke sahiptir.[15] Trke ok geni +kullanmyla birlikte zengin bir dil olmasnn yan sra, genel itibaryla +"zne-nesne-yklem" biimindeki cmle kuruluuna sahiptir. diff --git a/src/tests/testdata/tr/iso-8859-9.txt b/src/tests/testdata/tr/iso-8859-9.txt new file mode 100644 index 0000000..4a69aa3 --- /dev/null +++ b/src/tests/testdata/tr/iso-8859-9.txt @@ -0,0 +1,13 @@ +Trke, Trk dili ya da Trkiye Trkesi, batda Balkanlar'dan balayp douda +Hazar Denizi sahasna kadar konuulan Altay dillerinden biridir. Ya, en eski +hesaplara gre 8500 olan Trke, bugn yaayan Dnya dilleri arasnda en eski +yazl belgelere sahip olan dildir. Bu belgeler, ivi yazl Smerce +tabletlerdeki alnt kelimelerdir.[12] Trk yaz dilleri iinde Ouz sahas yaz +dillerinden Osmanl Trkesinin devamn oluturur. Bata Trkiye olmak zere +eski Osmanl mparatorluu corafyasnda konuulan Trke, dnyada en fazla +konuulan 5. dildir. Trke sondan eklemeli bir dildir.[13] Bundan tr +kullanlan herhangi bir eylem zerinden istenildii kadar szck +tretilebilir.[14] Trkiye Trkesi bu ynnden dolay dier Trk dilleriyle +ortak ya da ayrk bulunan onlarca eke sahiptir.[15] Trke ok geni +kullanmyla birlikte zengin bir dil olmasnn yan sra, genel itibaryla +"zne-nesne-yklem" biimindeki cmle kuruluuna sahiptir. diff --git a/src/tests/testdata/vi/utf-8.txt b/src/tests/testdata/vi/utf-8.txt new file mode 100644 index 0000000..c82798d --- /dev/null +++ b/src/tests/testdata/vi/utf-8.txt @@ -0,0 +1,4 @@ +Chữ Quốc ngữ là hệ chữ viết thống nhất chính thức hiện nay của tiếng Việt, sử +dụng ký tự La Tinh, dựa trên các bảng chữ cái của nhóm ngôn ngữ Rôman,[1] đặc +biệt là bảng chữ cái Bồ Đào Nha,[2] với các dấu phụ chủ yếu từ bảng chữ cái Hy +Lạp. diff --git a/src/tests/testdata/vi/viscii.txt b/src/tests/testdata/vi/viscii.txt new file mode 100644 index 0000000..ef1e187 --- /dev/null +++ b/src/tests/testdata/vi/viscii.txt @@ -0,0 +1,4 @@ +Ch Quc ng l h ch vit thng nht chnh thc hin nay ca ting Vit, s +dng k t La Tinh, da trn cc bng ch ci ca nhm ngn ng Rman,[1] c +bit l bng ch ci B o Nha,[2] vi cc du ph ch yu t bng ch ci Hy +Lp. diff --git a/src/tests/testdata/vi/windows-1258.txt b/src/tests/testdata/vi/windows-1258.txt new file mode 100644 index 0000000..53526db --- /dev/null +++ b/src/tests/testdata/vi/windows-1258.txt @@ -0,0 +1,4 @@ +Ch Quc ng l h ch vit thng nht chnh thc hin nay cua ting Vit, s +dung ky t La Tinh, da trn cc bang ch ci cua nhm ngn ng Rman,[1] c +bit l bang ch ci B o Nha,[2] vi cc du phu chu yu t bang ch ci Hy +Lap. diff --git a/src/tests/testdata/zh/GB18030/wikitop_zh_GB18030.txt b/src/tests/testdata/zh/GB18030/wikitop_zh_GB18030.txt deleted file mode 100644 index 18fefe6..0000000 --- a/src/tests/testdata/zh/GB18030/wikitop_zh_GB18030.txt +++ /dev/null @@ -1,251 +0,0 @@ -ٿ:ҳ - - άٿƣɵİٿȫ - ת , - - Ϲ \{ɫ - {ɫ - ɫ - ɫ - ӆf - - - {ٴ˴ - άٿ - ˿ɾ݋ɰٿȫ - 381,540ƪėlĿ - - * gӭ - * - * - * } - - * ɫ - * ΰ - * ı - * ֙CSٿ - - - -ɫĿ - - Statue of Silent Witness.jpg - - Ӣ󎟣һƥۼNِRFѽۡӢ󎟳춰ޣڶ;G;\ˡLo;ϯ;ȵِŠZڣʷ - OּƶƥB17Rƥжِxۼo䛡2004꣬ӢԸِRƥxrsIȫӰ - ˡ֮һRɸ|Ӗِ¾T־T20072½һِۣڰīһꡣӢ󎟷 - gِRR頭پWվ頭lж헼oƷKu۱lŮʰӢ - 桷DVDӰ - - Ŀ̨н - ij -  - h̖ĸŞ - - ѡ | 浵 | ɫĿ... - - -Ŷ̬ - - ն̹- - * 1023գ䷲ʡ7.2ǿҵ85500ˡ - * 1022գɳذն̹-ȣͼ֢ŦԼȥ83ꡣ - * 1021գŷ٤Զλϵͳڷ̫ɶ˹˺ػա - * 1020գ˹˷֯֯÷װ - * 1020գǰǸ쵼°ն֮սбȫίԱ쵼ĹžС - - S„ | | ... - - -Ŀ - - Knez Mihailo, Republic Square.jpg - - ά׶ijУλάDZߺӺͶ觺ӻϴ˴ҲŵƽԭͰͶɰ뵺2002꣬¹ - ˿1,576,124ԭ˹ijУҲǽ˹̹ŵͲ˹صĶŷĴСµס - ڹԪǰ4800²ʱڡ1284꣬µһγΪά˹ķ׶˺ȺΪάǡ˹׶ԼS - ͺɽ׶άΨһֱϽУΪ17УÿһӵԼĵطίԱᡣάǵľáĻ - ͿƼġ - - x | 浵 | ЃlĿ... - - - -֪ - - * ӡȶЩģD - * ķN|ܽwS׷Q׻w? - * 1936꽨ʱΪ߽ - * ǶЩģ - * λԱй̳һõھ? - * 龰ϲ硶ú롷ĵڶЩ缯 - - ѡ | 浵 | Ŀ | Ŀ... - - -ʷϵĽ - - 1024գϹ - ɳض - * 1260꣬ɳضãͼɣ·׾ʽ - * 1648꣬ʥ۹ͷȹ˹ʽȷϡ˹طǺԼʮս - * 1860꣬й峯DӢ𲮾ǩԼuøӢ - * 1929꣬ŦԼ֤ȯɼ۱ȫ򾭼ô - * 1945꣬Ϲ¡ڻӢйͷȶǩֹ׼ЧϹ - - | 浵 | ʷ¼... - - - -ÿͼƬ - - Teide qtl1.jpg - - ̩·ʹ߷壬ϵɽһɽλڼȺѵҲǼȺĵرꡣɽ - Χ̩¹ҹ԰ռ18900꣬2007629ձΪŲ2008̩꣬¹ҹ԰ǼȺοĹҹ԰ - 2004꣬350οͣοձʿɽĻɽ̩·庣θ߶Ϊ3718ףʹߵɽ塣ͼΪ̩·塣 - - ÿͼƬ | ɫͼƬ - - - - άٿһɡκ˶ܲ롢жԵİٿȫЭƻǵĿǽһ׼ȷİٿȫ顣 - - CC-BY-SA-3.0Э·κ˶ڸЭҪʹЩݡ - - ֲ | ߽άٿ - - - κ˶޸άٿϵݣǵķָ - - ǴĿDZ༭ĿΪĿһͼΪάٿס - - ༭ֲ | FAQ - - - ǰ1,076,572עûлԾû5,864Щ־ԸͨջҳкۡҲIRCʼ - йͨ - - ҳ - - - Sٿ㡭 - - @猦҂JR - -ǷȥSٿƸе棿㿴҂Wikipedia:ŽıűάٿƵݡWikipedia as - a press source -(ӢľS)֪Sٿѽu„עҕҾSٿƵăݽ„硢硢WgǷͥYӍԴ֪SٿھWj -ĸrԅAlexag[桢UsenetGoogle„𰸲ԃա - - ԔҊ: Wikipedia:άٿý - - һʾ - һʾ - - ʾȫ - һSCʾ - ʾϷ - - - άٿɷӪ֯άýԡݵľSƻ - Meta Wiki - Ԫά - Эάƻ - wiktionary - άʵ - ֵʹʵ - wikinews - ά - ɵԴ - wikibooks - ά̿ - ֲ̿ - wikiquote - ά¼ - Եļ - Wikisource - άĿ - ɵͼ - Wikispecies - ά - ɵϿ - wikiversity - άѧԺ - ɵϰȺ - Commons - άԴ - ĶýϿ - - ھSٿ࣬Ո]YُO䡣cİȡ“MՈ„M] - If you do not write Chinese but need to contact zh-wikipedia, you can - leave your message here. - - ԡhttp://zh.wikipedia.org/w/index.php?title=Wikipedia:%E9%A6%96%E9%A1% - B5&oldid=18000640 - -˹ - - * ¼/˻ - -ֿռ - - * Ŀҳ - * - -ת - -任 - - * ת - * - * w - * ½ - * ۰ķw - * ¼ - * ̨w - -鿴 - - * Ķ - * 鿴Դ - * 鿴ʷ - - - - - - ____________________ (Submit) - - - - * ҳ - * - * ɫ - * Ŷ̬ - * - * Ŀ - - - - * - * άȺ - * ָ - * ջ - * ѯʴ - * ִת - * IRCʱ - * ϵ - * άٿ - * άٿ - - - - * ҳ - * - * ϴļ - * ҳ - * ӡҳ - * - - - - * ҳ޶20111010 (һ) 16:03 - * վȫ֪ʶ -ͬʽ 3.0Э֮ṩӦáʹ - Wikipedia03άٿƱ־άýע̱ꣻά64άý̱ꡣ - άýݵǼǵ501(c)(3)˰Ӫƻ - - * ˽ - * άٿ - * - * ƶ - diff --git a/src/tests/testdata/zh/UTF-8/wikitop_zh_UTF-8.txt b/src/tests/testdata/zh/UTF-8/wikitop_zh_UTF-8.txt deleted file mode 100644 index 8458876..0000000 --- a/src/tests/testdata/zh/UTF-8/wikitop_zh_UTF-8.txt +++ /dev/null @@ -1,251 +0,0 @@ -基百科:首页 - - 维基百科，自由的百科全书 - 跳转到： 导航, 搜索 - - 联合供稿 淺藍色 - 深藍色 - 灰色 - 橙色 - 自訂首頁說明 - - - 海納百川，有容乃大 - 维基百科 - 人人可編輯的自由百科全書 - 已有381,540篇中文條目 - - * · 歡迎 - * · 內容 - * · 分類 - * · 主題 - - * · 特色內容 - * · 簡單版首頁 - * · 纯文本首頁 - * · 手機版維基百科 - - - -特色条目 - - Statue of Silent Witness.jpg - - 精英大師，是一匹香港純種賽馬，現在已經退役。精英大師出生於澳洲，在短途三冠洋紫荊短途錦標、百週年紀念短途盃、主席短途獎等等賽事奪冠，創下世界史 - 上設立分級制度後首匹連勝17場的馬匹，另有多項香港賽駒的香港紀錄。2004年，精英大師以競賽馬匹的身份入選美國《時代》雜誌年度全球最具影響力「 - 人」物之一。此馬由告東尼訓練，所有賽事均由騎師高雅志策騎。2007年2月角逐最後一場賽事後退役，在澳洲墨爾本近郊的一個牧場安享晚年。精英大師服 - 役期間，被香港賽馬會大力宣傳，馬會為牠建立專屬網站，亦為牠發行多項紀念品，並且製作《愛彼錶女皇盃及精英大師 - 不敗傳奇》DVD影碟。 - - 其他条目：台灣國中教育 - 夏朝 - 天津碱厂 - 漢考克號航空母艦 - - 候选 | 存档 | 所有特色条目... - - -新闻动态 - - 苏尔坦·本·阿卜杜勒-阿齐兹 - * 10月23日，土耳其凡城省发生里氏7.2级强烈地震，造成至少85人死亡、500多人受伤。 - * 10月22日，沙特阿拉伯王储苏尔坦·本·阿卜杜勒-阿齐兹（图）因癌症在美国纽约去世，享年83岁。 - * 10月21日，欧洲伽利略定位系统首批两颗卫星在法属圭亚那太空中心由俄罗斯联盟号运载火箭搭载升空。 - * 10月20日，西班牙巴斯克人分裂组织埃塔宣布该组织将永久放弃武装斗争。 - * 10月20日，前利比亚革命领导人穆阿迈尔·卡扎菲在苏尔特之战中被全国过渡委员会领导的国民解放军击毙。 - - 維基新聞 | 讣闻 | 更多新闻... - - -优良条目 - - Knez Mihailo, Republic Square.jpg - - 贝尔格莱德是塞尔维亚首都和最大的城市，位于塞尔维亚北部萨瓦河和多瑙河汇合处，此处也是潘诺尼亚平原和巴尔干半岛的相遇处。2002年，贝尔格莱德共 - 有人口1,576,124，是原南斯拉夫地区最大的城市，也是仅次于伊斯坦布尔、雅典和布加勒斯特的东南欧第四大城市。贝尔格莱德地区最早的人类居住出 - 现在公元前4800年温查文明时期。1284年，贝尔格莱德第一次成为塞尔维亚斯雷姆王国的首都。此后，它还先后成为塞尔维亚、南斯拉夫首都以及塞爾維 - 亞和黑山的行政、立法首都。贝尔格莱德是塞尔维亚唯一的直辖市，被分为17个自治市，每一个都拥有自己的地方委员会。贝尔格莱德是塞尔维亚的经济、文化 - 、教育和科技中心。 - - 候選 | 存档 | 所有優良條目... - - - -你知道吗？ - - * 印度都有哪些旗帜？（圖） - * 哪種物質可溶解硝化纖維，俗稱二甲基溶纖劑? - * 哪座建筑在1936年建成时为天津市最高建筑？ - * 马来西亚都有哪些旗帜？ - * 哪位球员是中国足坛第一个拿到国内三个级别联赛冠军的人? - * 美国情景喜剧《好汉两个半》的第二季都有哪些剧集？ - - 候选 | 存档 | 创建新条目 | 更多新条目... - - -历史上的今天 - - 10月24日：联合国日 - 沙特尔大教堂 - * 1260年，法国沙特尔大教堂（图）落成，法国国王路易九世亲临落成仪式。 - * 1648年，神圣罗马帝国和法国、瑞典等国代表在明斯特正式确认《威斯特伐利亚和约》，三十年战争结束。 - * 1860年，中国清朝代表奕訢与英国代表额尔金伯爵签订《北京条约》，將九龍半島割让给英国。 - * 1929年，美国纽约证券交易所股价暴跌，引发全球经济大萧条。 - * 1945年，《联合国宪章》在获得美国、英国、中国、苏联和法国等多数签字国的批准后生效，联合国成立。 - - 节日 | 存档 | 更多历史事件... - - - -每日图片 - - Teide qtl1.jpg - - 泰德峰是西班牙和大西洋岛屿的最高峰，并且是世界上第三大火山。这是一座活火山，位于加那利群岛的特内里费岛，也是加那利群岛最著名的地标。火山及其周 - 围组成了泰德国家公园，占地18900公顷，2007年6月29日被列为世界遗产。2008年，泰德国家公园是加那利群岛和西班牙游客最多的国家公园。 - 2004年，有350万游客，是游客人数仅次于日本富士山的火山。泰德峰海拔高度为3718米，是西班牙和大西洋中最高的山峰。图为泰德峰。 - - 每日图片 | 特色图片 - - - - 维基百科是一个内容自由、任何人都能参与、并有多种语言的百科全书协作计划。我们的目标是建立一个完整、准确和中立的百科全书。 - - 所有文字内容在CC-BY-SA-3.0协议下发布，任何人都可以在该协议条款的要求下自由使用这些内容。 - - 读者手册 | 走进维基百科 - - - 任何人都可以修改维基百科上的内容，但是请遵守我们的方针和指引。 - - 无论是创建新条目，还是编辑现有条目，或者为条目增加一幅插图，您都可以为维基百科作出贡献。 - - 编辑手册 | FAQ - - - 当前有1,076,572名注册用户，其中活跃用户有5,864名。所有这些志愿者通过互助客栈和讨论页进行合作与讨论。或者您也可以用IRC或邮件列 - 表进行沟通。 - - 社区主页 - - - 維基百科提醒你… - - 這個世界對我們的認識 - -是否對於世界各地怎去看維基百科感到好奇？若你看看我們的Wikipedia:新闻界的报导、新闻报导引用维基百科的内容、Wikipedia as - a press source -(英文維基)，你會知道維基百科已經逐漸被新聞界注視，且維基百科的內容經常作為新聞界、出版界、學術界甚至是法庭的資訊來源。欲知道維基百科在網絡世 -界的競爭狀況，可以參考Alexa瀏覽量報告、Usenet公告板和Google的新聞及答案查詢服務。 - - 詳細另見: Wikipedia:维基百科与媒体 - - 上一個提示 - 下一個提示 - - 提示大全 - 下一個隨機提示 - 回提示最上方 - - - 维基百科由非营利组织维基媒体基金会运作。基金会旗下尚有其他數个多语言、自由内容的維基计划： - Meta Wiki - 元维基 - 协调各维基计划 - wiktionary - 维基词典 - 多语言字典和词典 - wikinews - 维基新闻 - 自由的新闻资源 - wikibooks - 维基教科书 - 教科书和手册 - wikiquote - 维基语录 - 名人名言的集锦 - Wikisource - 维基文库 - 自由的图书馆 - Wikispecies - 维基物种 - 自由的物种资料库 - wikiversity - 维基学院 - 自由的研习社群 - Commons - 维基共享资源 - 共享的多媒体资料库 - - 若您在維基百科受益良多，請考慮資助基金會添購設備。欲與中文版取得聯繫，請至新聞組或寄送電子郵件。 - If you do not write Chinese but need to contact zh-wikipedia, you can - leave your message here. - - 来自“http://zh.wikipedia.org/w/index.php?title=Wikipedia:%E9%A6%96%E9%A1% - B5&oldid=18000640” - -个人工具 - - * 登录/创建账户 - -名字空间 - - * 项目页面 - * 讨论 - -不转换 - -变换 - - * 不转换 - * 简体 - * 繁體 - * 大陆简体 - * 港澳繁體 - * 马新简体 - * 台灣正體 - -查看 - - * 阅读 - * 查看源代码 - * 查看历史 - -操作 - -搜索 - - ____________________ (Submit) 搜索 - -导航 - - * 首页 - * 分類索引 - * 特色内容 - * 新闻动态 - * 最近更改 - * 随机条目 - -帮助 - - * 帮助 - * 维基社群 - * 方针与指引 - * 互助客栈 - * 询问处 - * 字词转换 - * IRC即时聊天 - * 联系我们 - * 关于维基百科 - * 资助维基百科 - -工具 - - * 链入页面 - * 链出更改 - * 上传文件 - * 特殊页面 - * 打印页面 - * 永久链接 - -其他语言 - - * 本页面最后修订于2011年10月10日 (星期一) 16:03。 - * 本站的全部文字在知识共享 署名-相同方式共享 3.0协议之条款下提供，附加条款亦可能应用。（请参阅使用条款） - Wikipedia®和维基百科标志是维基媒体基金会的注册商标；维基™是维基媒体基金会的商标。 - 维基媒体基金会是在美国佛罗里达州登记的501(c)(3)免税、非营利、慈善机构。 - - * 隐私政策 - * 关于维基百科 - * 免责声明 - * 移动浏览 - diff --git a/src/tests/testdata/zh/big5.txt b/src/tests/testdata/zh/big5.txt new file mode 100644 index 0000000..59db954 --- /dev/null +++ b/src/tests/testdata/zh/big5.txt @@ -0,0 +1 @@ +c餤c餤c餤c餤c餤c餤c餤c餤c餤c餤c餤c餤c餤c餤c餤c餤c餤 \ No newline at end of file diff --git a/src/tests/testdata/zh/euc-tw.txt b/src/tests/testdata/zh/euc-tw.txt new file mode 100644 index 0000000..ba30a9a --- /dev/null +++ b/src/tests/testdata/zh/euc-tw.txt @@ -0,0 +1 @@ +EUC-TWҳƺġCNS 11643??ƺEUC-TWŷ diff --git a/src/tests/testdata/zh/gb18030.txt b/src/tests/testdata/zh/gb18030.txt new file mode 100644 index 0000000..962df87 --- /dev/null +++ b/src/tests/testdata/zh/gb18030.txt @@ -0,0 +1 @@ +ļļļļļļļļļļ \ No newline at end of file diff --git a/src/tests/testdata/zh/utf-8.txt b/src/tests/testdata/zh/utf-8.txt new file mode 100644 index 0000000..cd66b08 --- /dev/null +++ b/src/tests/testdata/zh/utf-8.txt @@ -0,0 +1 @@ +汉字漢字統一編碼萬國碼