001////////////////////////////////////////////////////////////////////////////////
002// checkstyle: Checks Java source code for adherence to a set of rules.
003// Copyright (C) 2001-2016 the original author or authors.
004//
005// This library is free software; you can redistribute it and/or
006// modify it under the terms of the GNU Lesser General Public
007// License as published by the Free Software Foundation; either
008// version 2.1 of the License, or (at your option) any later version.
009//
010// This library is distributed in the hope that it will be useful,
011// but WITHOUT ANY WARRANTY; without even the implied warranty of
012// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
013// Lesser General Public License for more details.
014//
015// You should have received a copy of the GNU Lesser General Public
016// License along with this library; if not, write to the Free Software
017// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
018////////////////////////////////////////////////////////////////////////////////
019
020package com.puppycrawl.tools.checkstyle.checks;
021
022import java.io.File;
023import java.io.FileInputStream;
024import java.io.FileNotFoundException;
025import java.io.IOException;
026import java.io.InputStream;
027import java.util.Collections;
028import java.util.List;
029import java.util.Locale;
030import java.util.Optional;
031import java.util.Properties;
032import java.util.Set;
033import java.util.SortedSet;
034import java.util.regex.Matcher;
035import java.util.regex.Pattern;
036
037import org.apache.commons.logging.Log;
038import org.apache.commons.logging.LogFactory;
039
040import com.google.common.collect.HashMultimap;
041import com.google.common.collect.SetMultimap;
042import com.google.common.collect.Sets;
043import com.google.common.io.Closeables;
044import com.google.common.io.Files;
045import com.puppycrawl.tools.checkstyle.Definitions;
046import com.puppycrawl.tools.checkstyle.api.AbstractFileSetCheck;
047import com.puppycrawl.tools.checkstyle.api.LocalizedMessage;
048import com.puppycrawl.tools.checkstyle.api.MessageDispatcher;
049import com.puppycrawl.tools.checkstyle.utils.CommonUtils;
050
051/**
052 * <p>
053 * The TranslationCheck class helps to ensure the correct translation of code by
054 * checking locale-specific resource files for consistency regarding their keys.
055 * Two locale-specific resource files describing one and the same context are consistent if they
056 * contain the same keys. TranslationCheck also can check an existence of required translations
057 * which must exist in project, if 'requiredTranslations' option is used.
058 * </p>
059 * <p>
060 * An example of how to configure the check is:
061 * </p>
062 * <pre>
063 * &lt;module name="Translation"/&gt;
064 * </pre>
065 * Check has the following options:
066 *
067 * <p><b>baseName</b> - a base name regexp for resource bundles which contain message resources. It
068 * helps the check to distinguish config and localization resources. Default value is
069 * <b>^messages.*$</b>
070 * <p>An example of how to configure the check to validate only bundles which base names start with
071 * "ButtonLabels":
072 * </p>
073 * <pre>
074 * &lt;module name="Translation"&gt;
075 *     &lt;property name="baseName" value="^ButtonLabels.*$"/&gt;
076 * &lt;/module&gt;
077 * </pre>
078 * <p>To configure the check to check only files which have '.properties' and '.translations'
079 * extensions:
080 * </p>
081 * <pre>
082 * &lt;module name="Translation"&gt;
083 *     &lt;property name="fileExtensions" value="properties, translations"/&gt;
084 * &lt;/module&gt;
085 * </pre>
086 *
087 * <p><b>requiredTranslations</b> which allows to specify language codes of required translations
088 * which must exist in project. Language code is composed of the lowercase, two-letter codes as
089 * defined by <a href="https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">ISO 639-1</a>.
090 * Default value is <b>empty String Set</b> which means that only the existence of
091 * default translation is checked. Note, if you specify language codes (or just one language
092 * code) of required translations the check will also check for existence of default translation
093 * files in project. ATTENTION: the check will perform the validation of ISO codes if the option
094 * is used. So, if you specify, for example, "mm" for language code, TranslationCheck will rise
095 * violation that the language code is incorrect.
096 * <br>
097 *
098 * @author Alexandra Bunge
099 * @author lkuehne
100 * @author Andrei Selkin
101 */
102public class TranslationCheck extends AbstractFileSetCheck {
103
104    /**
105     * A key is pointing to the warning message text for missing key
106     * in "messages.properties" file.
107     */
108    public static final String MSG_KEY = "translation.missingKey";
109
110    /**
111     * A key is pointing to the warning message text for missing translation file
112     * in "messages.properties" file.
113     */
114    public static final String MSG_KEY_MISSING_TRANSLATION_FILE =
115        "translation.missingTranslationFile";
116
117    /** Resource bundle which contains messages for TranslationCheck. */
118    private static final String TRANSLATION_BUNDLE =
119        "com.puppycrawl.tools.checkstyle.checks.messages";
120
121    /**
122     * A key is pointing to the warning message text for wrong language code
123     * in "messages.properties" file.
124     */
125    private static final String WRONG_LANGUAGE_CODE_KEY = "translation.wrongLanguageCode";
126
127    /** Logger for TranslationCheck. */
128    private static final Log LOG = LogFactory.getLog(TranslationCheck.class);
129
130    /**
131     * Regexp string for default tranlsation files.
132     * For example, messages.properties.
133     */
134    private static final String DEFAULT_TRANSLATION_REGEXP = "^.+\\..+$";
135
136    /**
137     * Regexp pattern for bundles names wich end with language code, followed by country code and
138     * variant suffix. For example, messages_es_ES_UNIX.properties.
139     */
140    private static final Pattern LANGUAGE_COUNTRY_VARIANT_PATTERN =
141        CommonUtils.createPattern("^.+\\_[a-z]{2}\\_[A-Z]{2}\\_[A-Za-z]+\\..+$");
142    /**
143     * Regexp pattern for bundles names wich end with language code, followed by country code
144     * suffix. For example, messages_es_ES.properties.
145     */
146    private static final Pattern LANGUAGE_COUNTRY_PATTERN =
147        CommonUtils.createPattern("^.+\\_[a-z]{2}\\_[A-Z]{2}\\..+$");
148    /**
149     * Regexp pattern for bundles names wich end with language code suffix.
150     * For example, messages_es.properties.
151     */
152    private static final Pattern LANGUAGE_PATTERN =
153        CommonUtils.createPattern("^.+\\_[a-z]{2}\\..+$");
154
155    /** File name format for default translation. */
156    private static final String DEFAULT_TRANSLATION_FILE_NAME_FORMATTER = "%s.%s";
157    /** File name format with language code. */
158    private static final String FILE_NAME_WITH_LANGUAGE_CODE_FORMATTER = "%s_%s.%s";
159
160    /** Formatting string to form regexp to validate required tranlsations file names. */
161    private static final String REGEXP_FORMAT_TO_CHECK_REQUIRED_TRANSLATIONS =
162        "^%1$s\\_%2$s(\\_[A-Z]{2})?\\.%3$s$|^%1$s\\_%2$s\\_[A-Z]{2}\\_[A-Za-z]+\\.%3$s$";
163    /** Formatting string to form regexp to validate default tranlsations file names. */
164    private static final String REGEXP_FORMAT_TO_CHECK_DEFAULT_TRANSLATIONS = "^%s\\.%s$";
165
166    /** The files to process. */
167    private final Set<File> filesToProcess = Sets.newHashSet();
168
169    /** The base name regexp pattern. */
170    private Pattern baseNamePattern;
171
172    /**
173     * Language codes of required translations for the check (de, pt, ja, etc).
174     */
175    private Set<String> requiredTranslations = Sets.newHashSet();
176
177    /**
178     * Creates a new {@code TranslationCheck} instance.
179     */
180    public TranslationCheck() {
181        setFileExtensions("properties");
182        baseNamePattern = CommonUtils.createPattern("^messages.*$");
183    }
184
185    /**
186     * Sets the base name regexp pattern.
187     * @param baseName base name regexp.
188     */
189    public void setBaseName(String baseName) {
190        baseNamePattern = CommonUtils.createPattern(baseName);
191    }
192
193    /**
194     * Sets language codes of required translations for the check.
195     * @param translationCodes a comma separated list of language codes.
196     */
197    public void setRequiredTranslations(String... translationCodes) {
198        requiredTranslations = Sets.newHashSet(translationCodes);
199        validateUserSpecifiedLanguageCodes(requiredTranslations);
200    }
201
202    /**
203     * Validates the correctness of user specififed language codes for the check.
204     * @param languageCodes user specified language codes for the check.
205     */
206    private void validateUserSpecifiedLanguageCodes(Set<String> languageCodes) {
207        for (String code : languageCodes) {
208            if (!isValidLanguageCode(code)) {
209                final LocalizedMessage msg = new LocalizedMessage(0, TRANSLATION_BUNDLE,
210                        WRONG_LANGUAGE_CODE_KEY, new Object[] {code}, getId(), getClass(), null);
211                final String exceptionMessage = String.format(Locale.ROOT,
212                        "%s [%s]", msg.getMessage(), TranslationCheck.class.getSimpleName());
213                throw new IllegalArgumentException(exceptionMessage);
214            }
215        }
216    }
217
218    /**
219     * Checks whether user specified language code is correct (is contained in available locales).
220     * @param userSpecifiedLanguageCode user specified language code.
221     * @return true if user specified language code is correct.
222     */
223    private static boolean isValidLanguageCode(final String userSpecifiedLanguageCode) {
224        boolean valid = false;
225        final Locale[] locales = Locale.getAvailableLocales();
226        for (Locale locale : locales) {
227            if (userSpecifiedLanguageCode.equals(locale.toString())) {
228                valid = true;
229                break;
230            }
231        }
232        return valid;
233    }
234
235    @Override
236    public void beginProcessing(String charset) {
237        super.beginProcessing(charset);
238        filesToProcess.clear();
239    }
240
241    @Override
242    protected void processFiltered(File file, List<String> lines) {
243        // We just collecting files for processing at finishProcessing()
244        filesToProcess.add(file);
245    }
246
247    @Override
248    public void finishProcessing() {
249        super.finishProcessing();
250
251        final Set<ResourceBundle> bundles = groupFilesIntoBundles(filesToProcess, baseNamePattern);
252        for (ResourceBundle currentBundle : bundles) {
253            checkExistenceOfDefaultTranslation(currentBundle);
254            checkExistenceOfRequiredTranslations(currentBundle);
255            checkTranslationKeys(currentBundle);
256        }
257    }
258
259    /**
260     * Checks an existence of default translation file in the resource bundle.
261     * @param bundle resource bundle.
262     */
263    private void checkExistenceOfDefaultTranslation(ResourceBundle bundle) {
264        final Optional<String> fileName = getMissingFileName(bundle, null);
265        if (fileName.isPresent()) {
266            logMissingTranslation(bundle.getPath(), fileName.get());
267        }
268    }
269
270    /**
271     * Checks an existence of translation files in the resource bundle.
272     * The name of translation file begins with the base name of resource bundle which is followed
273     * by '_' and a language code (country and variant are optional), it ends with the extension
274     * suffix.
275     * @param bundle resource bundle.
276     */
277    private void checkExistenceOfRequiredTranslations(ResourceBundle bundle) {
278        for (String languageCode : requiredTranslations) {
279            final Optional<String> fileName = getMissingFileName(bundle, languageCode);
280            if (fileName.isPresent()) {
281                logMissingTranslation(bundle.getPath(), fileName.get());
282            }
283        }
284    }
285
286    /**
287     * Returns the name of translation file which is absent in resource bundle or Guava's Optional,
288     * if there is not missing translation.
289     * @param bundle resource bundle.
290     * @param languageCode language code.
291     * @return the name of translation file which is absent in resource bundle or Guava's Optional,
292     *         if there is not missing translation.
293     */
294    private static Optional<String> getMissingFileName(ResourceBundle bundle, String languageCode) {
295        final String fileNameRegexp;
296        final boolean searchForDefaultTranslation;
297        final String extension = bundle.getExtension();
298        final String baseName = bundle.getBaseName();
299        if (languageCode == null) {
300            searchForDefaultTranslation = true;
301            fileNameRegexp = String.format(Locale.ROOT, REGEXP_FORMAT_TO_CHECK_DEFAULT_TRANSLATIONS,
302                    baseName, extension);
303        }
304        else {
305            searchForDefaultTranslation = false;
306            fileNameRegexp = String.format(Locale.ROOT,
307                REGEXP_FORMAT_TO_CHECK_REQUIRED_TRANSLATIONS, baseName, languageCode, extension);
308        }
309        Optional<String> missingFileName = Optional.empty();
310        if (!bundle.containsFile(fileNameRegexp)) {
311            if (searchForDefaultTranslation) {
312                missingFileName = Optional.of(String.format(Locale.ROOT,
313                        DEFAULT_TRANSLATION_FILE_NAME_FORMATTER, baseName, extension));
314            }
315            else {
316                missingFileName = Optional.of(String.format(Locale.ROOT,
317                        FILE_NAME_WITH_LANGUAGE_CODE_FORMATTER, baseName, languageCode, extension));
318            }
319        }
320        return missingFileName;
321    }
322
323    /**
324     * Logs that translation file is missing.
325     * @param filePath file path.
326     * @param fileName file name.
327     */
328    private void logMissingTranslation(String filePath, String fileName) {
329        final MessageDispatcher dispatcher = getMessageDispatcher();
330        dispatcher.fireFileStarted(filePath);
331        log(0, MSG_KEY_MISSING_TRANSLATION_FILE, fileName);
332        fireErrors(filePath);
333        dispatcher.fireFileFinished(filePath);
334    }
335
336    /**
337     * Groups a set of files into bundles.
338     * Only files, which names match base name regexp pattern will be grouped.
339     * @param files set of files.
340     * @param baseNameRegexp base name regexp pattern.
341     * @return set of ResourceBundles.
342     */
343    private static Set<ResourceBundle> groupFilesIntoBundles(Set<File> files,
344                                                             Pattern baseNameRegexp) {
345        final Set<ResourceBundle> resourceBundles = Sets.newHashSet();
346        for (File currentFile : files) {
347            final String fileName = currentFile.getName();
348            final String baseName = extractBaseName(fileName);
349            final Matcher baseNameMatcher = baseNameRegexp.matcher(baseName);
350            if (baseNameMatcher.matches()) {
351                final String extension = Files.getFileExtension(fileName);
352                final String path = getPath(currentFile.getAbsolutePath());
353                final ResourceBundle newBundle = new ResourceBundle(baseName, path, extension);
354                final Optional<ResourceBundle> bundle = findBundle(resourceBundles, newBundle);
355                if (bundle.isPresent()) {
356                    bundle.get().addFile(currentFile);
357                }
358                else {
359                    newBundle.addFile(currentFile);
360                    resourceBundles.add(newBundle);
361                }
362            }
363        }
364        return resourceBundles;
365    }
366
367    /**
368     * Searches for specific resource bundle in a set of resource bundles.
369     * @param bundles set of resource bundles.
370     * @param targetBundle target bundle to search for.
371     * @return Guava's Optional of resource bundle (present if target bundle is found).
372     */
373    private static Optional<ResourceBundle> findBundle(Set<ResourceBundle> bundles,
374                                                       ResourceBundle targetBundle) {
375        Optional<ResourceBundle> result = Optional.empty();
376        for (ResourceBundle currentBundle : bundles) {
377            if (targetBundle.getBaseName().equals(currentBundle.getBaseName())
378                    && targetBundle.getExtension().equals(currentBundle.getExtension())
379                    && targetBundle.getPath().equals(currentBundle.getPath())) {
380                result = Optional.of(currentBundle);
381                break;
382            }
383        }
384        return result;
385    }
386
387    /**
388     * Extracts the base name (the unique prefix) of resource bundle from translation file name.
389     * For example "messages" is the base name of "messages.properties",
390     * "messages_de_AT.properties", "messages_en.properties", etc.
391     * @param fileName the fully qualified name of the translation file.
392     * @return the extracted base name.
393     */
394    private static String extractBaseName(String fileName) {
395        final String regexp;
396        final Matcher languageCountryVariantMatcher =
397            LANGUAGE_COUNTRY_VARIANT_PATTERN.matcher(fileName);
398        final Matcher languageCountryMatcher = LANGUAGE_COUNTRY_PATTERN.matcher(fileName);
399        final Matcher languageMatcher = LANGUAGE_PATTERN.matcher(fileName);
400        if (languageCountryVariantMatcher.matches()) {
401            regexp = LANGUAGE_COUNTRY_VARIANT_PATTERN.pattern();
402        }
403        else if (languageCountryMatcher.matches()) {
404            regexp = LANGUAGE_COUNTRY_PATTERN.pattern();
405        }
406        else if (languageMatcher.matches()) {
407            regexp = LANGUAGE_PATTERN.pattern();
408        }
409        else {
410            regexp = DEFAULT_TRANSLATION_REGEXP;
411        }
412        // We use substring(...) insead of replace(...), so that the regular expression does
413        // not have to be compiled each time it is used inside 'replace' method.
414        final String removePattern = regexp.substring("^.+".length(), regexp.length());
415        return fileName.replaceAll(removePattern, "");
416    }
417
418    /**
419     * Extracts path from a file name which contains the path.
420     * For example, if file nam is /xyz/messages.properties, then the method
421     * will return /xyz/.
422     * @param fileNameWithPath file name which contains the path.
423     * @return file path.
424     */
425    private static String getPath(String fileNameWithPath) {
426        return fileNameWithPath
427            .substring(0, fileNameWithPath.lastIndexOf(File.separator));
428    }
429
430    /**
431     * Checks resource files in bundle for consistency regarding their keys.
432     * All files in bundle must have the same key set. If this is not the case
433     * an error message is posted giving information which key misses in which file.
434     * @param bundle resource bundle.
435     */
436    private void checkTranslationKeys(ResourceBundle bundle) {
437        final Set<File> filesInBundle = bundle.getFiles();
438        if (filesInBundle.size() > 1) {
439            // build a map from files to the keys they contain
440            final Set<String> allTranslationKeys = Sets.newHashSet();
441            final SetMultimap<File, String> filesAssociatedWithKeys = HashMultimap.create();
442            for (File currentFile : filesInBundle) {
443                final Set<String> keysInCurrentFile = getTranslationKeys(currentFile);
444                allTranslationKeys.addAll(keysInCurrentFile);
445                filesAssociatedWithKeys.putAll(currentFile, keysInCurrentFile);
446            }
447            checkFilesForConsistencyRegardingTheirKeys(filesAssociatedWithKeys, allTranslationKeys);
448        }
449    }
450
451    /**
452     * Compares th the specified key set with the key sets of the given translation files (arranged
453     * in a map). All missing keys are reported.
454     * @param fileKeys a Map from translation files to their key sets.
455     * @param keysThatMustExist the set of keys to compare with.
456     */
457    private void checkFilesForConsistencyRegardingTheirKeys(SetMultimap<File, String> fileKeys,
458                                                            Set<String> keysThatMustExist) {
459        for (File currentFile : fileKeys.keySet()) {
460            final MessageDispatcher dispatcher = getMessageDispatcher();
461            final String path = currentFile.getPath();
462            dispatcher.fireFileStarted(path);
463            final Set<String> currentFileKeys = fileKeys.get(currentFile);
464            final Set<String> missingKeys = Sets.difference(keysThatMustExist, currentFileKeys);
465            if (!missingKeys.isEmpty()) {
466                for (Object key : missingKeys) {
467                    log(0, MSG_KEY, key);
468                }
469            }
470            fireErrors(path);
471            dispatcher.fireFileFinished(path);
472        }
473    }
474
475    /**
476     * Loads the keys from the specified translation file into a set.
477     * @param file translation file.
478     * @return a Set object which holds the loaded keys.
479     */
480    private Set<String> getTranslationKeys(File file) {
481        Set<String> keys = Sets.newHashSet();
482        InputStream inStream = null;
483        try {
484            inStream = new FileInputStream(file);
485            final Properties translations = new Properties();
486            translations.load(inStream);
487            keys = translations.stringPropertyNames();
488        }
489        catch (final IOException ex) {
490            logIoException(ex, file);
491        }
492        finally {
493            Closeables.closeQuietly(inStream);
494        }
495        return keys;
496    }
497
498    /**
499     * Helper method to log an io exception.
500     * @param exception the exception that occurred
501     * @param file the file that could not be processed
502     */
503    private void logIoException(IOException exception, File file) {
504        String[] args = null;
505        String key = "general.fileNotFound";
506        if (!(exception instanceof FileNotFoundException)) {
507            args = new String[] {exception.getMessage()};
508            key = "general.exception";
509        }
510        final LocalizedMessage message =
511            new LocalizedMessage(
512                0,
513                Definitions.CHECKSTYLE_BUNDLE,
514                key,
515                args,
516                getId(),
517                getClass(), null);
518        final SortedSet<LocalizedMessage> messages = Sets.newTreeSet();
519        messages.add(message);
520        getMessageDispatcher().fireErrors(file.getPath(), messages);
521        LOG.debug("IOException occurred.", exception);
522    }
523
524    /** Class which represents a resource bundle. */
525    private static class ResourceBundle {
526        /** Bundle base name. */
527        private final String baseName;
528        /** Common extension of files which are included in the resource bundle. */
529        private final String extension;
530        /** Common path of files which are included in the resource bundle. */
531        private final String path;
532        /** Set of files which are included in the resource bundle. */
533        private final Set<File> files;
534
535        /**
536         * Creates a ResourceBundle object with specific base name, common files extension.
537         * @param baseName bundle base name.
538         * @param path common path of files which are included in the resource bundle.
539         * @param extension common extension of files which are included in the resource bundle.
540         */
541        ResourceBundle(String baseName, String path, String extension) {
542            this.baseName = baseName;
543            this.path = path;
544            this.extension = extension;
545            files = Sets.newHashSet();
546        }
547
548        public String getBaseName() {
549            return baseName;
550        }
551
552        public String getPath() {
553            return path;
554        }
555
556        public String getExtension() {
557            return extension;
558        }
559
560        public Set<File> getFiles() {
561            return Collections.unmodifiableSet(files);
562        }
563
564        /**
565         * Adds a file into resource bundle.
566         * @param file file which should be added into resource bundle.
567         */
568        public void addFile(File file) {
569            files.add(file);
570        }
571
572        /**
573         * Checks whether a resource bundle contains a file which name matches file name regexp.
574         * @param fileNameRegexp file name regexp.
575         * @return true if a resource bundle contains a file which name matches file name regexp.
576         */
577        public boolean containsFile(String fileNameRegexp) {
578            boolean containsFile = false;
579            for (File currentFile : files) {
580                if (Pattern.matches(fileNameRegexp, currentFile.getName())) {
581                    containsFile = true;
582                    break;
583                }
584            }
585            return containsFile;
586        }
587    }
588}