'From Cuis 2.3 of 22 March 2010 [latest update: #472] on 15 April 2010 at 12:14:13 pm'!
!classDefinition: #Scanner category: #'Compiler-Kernel'!
Object subclass: #Scanner
	instanceVariableNames: 'source mark hereChar aheadChar token tokenType currentComment buffer typeTable isForTokenization '
	classVariableNames: 'TypeTable '
	poolDictionaries: ''
	category: 'Compiler-Kernel'!

!Scanner methodsFor: 'initialize-release' stamp: 'jmv 4/14/2010 11:20'!
initScanner

	buffer := WriteStream on: (String new: 40).
	typeTable := TypeTable.
	isForTokenization := false! !

!Scanner methodsFor: 'initialize-release' stamp: 'jmv 4/14/2010 11:24'!
initScannerForTokenization
	"Don't raise xIllegal when enocuntering an _"
	isForTokenization _ true! !

!Scanner methodsFor: 'multi-character scans' stamp: 'jmv 4/15/2010 12:14'!
xUnderscore
	self allowUnderscoreAssignments ifTrue:[ | type |
		"Figure out if x _foo (no space between _ and foo) 
		should be a selector or assignment"
		(((type := self typeTableAt: aheadChar) == #xLetter
			or:[type == #xDigit or:[type == #xUnderscore]]) 
			and:[self allowUnderscoreSelectors]) ifFalse:[
				self step.
				tokenType := #leftArrow.
				^token := #'_'
		].
	].
	self allowUnderscoreSelectors ifTrue:[^self xLetter].
	^isForTokenization
		ifFalse: [self xIllegal]
		ifTrue: [self xLetter]! !


!Parser methodsFor: 'private' stamp: 'jmv 4/14/2010 11:23'!
allowUnderscoreAssignments
	"Query class + preference"
	encoder == self ifTrue: [ ^super allowUnderscoreAssignments ].
	encoder ifNil: [ ^super allowUnderscoreAssignments ].
	^encoder classEncoding allowUnderscoreAssignments
		ifNil:[super allowUnderscoreAssignments]! !

!Parser methodsFor: 'private' stamp: 'jmv 4/14/2010 13:33'!
allowUnderscoreSelectors
	"Query class + preference"
	encoder == self ifTrue: [ ^super allowUnderscoreSelectors ].
	encoder ifNil: [ ^super allowUnderscoreSelectors ].
	^encoder classEncoding allowUnderscoreSelectors
		ifNil:[super allowUnderscoreSelectors]! !


!Scanner class methodsFor: 'testing' stamp: 'jmv 4/14/2010 09:43'!
isLiteralSymbol: aSymbol 
	"Test whether a symbol can be stored as # followed by its characters.  
	Symbols created internally with asSymbol may not have this property, 
	e.g. '3' asSymbol."
	| i ascii type |
	i := aSymbol size.
	i = 0 ifTrue: [^ false].
	ascii := (aSymbol at: 1) asciiValue.
	"TypeTable should have been origined at 0 rather than 1 ..."
	ascii = 0 ifTrue: [^ false].
	type := TypeTable at: ascii ifAbsent: [#xLetter].
	type == #xBinary ifTrue:
		[^i <= 2
		  and: [i = 1 or: [(TypeTable at: (aSymbol at: 2) asciiValue ifAbsent: []) == #xBinary]]].
	(type == #xColon or: [type == #verticalBar]) ifTrue:
		[^i = 1].
	(type == #xLetter or: [type == #xUnderscore and:[Preferences allowUnderscoreSelectors]]) ifTrue:
		[[i > 1] whileTrue:
			[ascii := (aSymbol at: i) asciiValue.
			ascii = 0 ifTrue: [^false].
			type := TypeTable at: ascii ifAbsent: [#xLetter].
			((type == #xLetter or: [type == #xUnderscore and:[Preferences allowUnderscoreSelectors]])or: [type == #xDigit or: [type == #xColon]]) ifFalse:
				[^false].
			i := i - 1].
		^true].
	^false! !

Scanner removeSelector: #xUnderscoreForTokenization!
!classDefinition: #Scanner category: #'Compiler-Kernel'!
Object subclass: #Scanner
	instanceVariableNames: 'source mark hereChar aheadChar token tokenType currentComment buffer typeTable isForTokenization'
	classVariableNames: 'TypeTable'
	poolDictionaries: ''
	category: 'Compiler-Kernel'!
