23 lines
529 B
JavaScript
23 lines
529 B
JavaScript
'use strict';
|
|
|
|
const index = require('../tokenizer/index.cjs');
|
|
|
|
const FUNCTION_TYPE = 'function';
|
|
|
|
/**
|
|
* Gets the tokenizer function from the configuration object or returns the default tokenizer
|
|
*
|
|
* @param config Configuration object
|
|
* @returns Corresponding tokenizer function
|
|
*/
|
|
function getTokenizer(config) {
|
|
if (config && typeof config.tokenize === FUNCTION_TYPE) {
|
|
return config.tokenize;
|
|
}
|
|
|
|
// Fallback to the default tokenizer
|
|
return index.tokenize;
|
|
}
|
|
|
|
exports.getTokenizer = getTokenizer;
|