1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
|
/* ANTLRTokenBuffer.h
*
* SOFTWARE RIGHTS
*
* We reserve no LEGAL rights to the Purdue Compiler Construction Tool
* Set (PCCTS) -- PCCTS is in the public domain. An individual or
* company may do whatever they wish with source code distributed with
* PCCTS or the code generated by PCCTS, including the incorporation of
* PCCTS, or its output, into commerical software.
*
* We encourage users to develop software with PCCTS. However, we do ask
* that credit is given to us for developing PCCTS. By "credit",
* we mean that if you incorporate our source code into one of your
* programs (commercial product, research project, or otherwise) that you
* acknowledge this fact somewhere in the documentation, research report,
* etc... If you like PCCTS and have developed a nice tool with the
* output, please mention that you developed it using PCCTS. In
* addition, we ask that this header remain intact in our source code.
* As long as these guidelines are kept, we expect to continue enhancing
* this system and expect to make other tools available as they are
* completed.
*
* ANTLR 1.33
* Terence Parr
* Parr Research Corporation
* with Purdue University and AHPCRC, University of Minnesota
* 1989-2000
*/
#ifndef ATOKENBUFFER_H_GATE
#define ATOKENBUFFER_H_GATE
#include "pcctscfg.h"
#include "pccts_stdlib.h"
PCCTS_NAMESPACE_STD
#include ATOKEN_H
#include ATOKENSTREAM_H
/*
* The parser is "attached" to an ANTLRTokenBuffer via interface
* functions: getToken() and bufferedToken(). The object that actually
* consumes characters and constructs tokens is connected to the
* ANTLRTokenBuffer via interface function ANTLRTokenStream::getToken();
* where ANTLRTokenStream is really just a behavior (class with no data).
* C++ does not have this abstraction and hence we simply have come up
* with a fancy name for "void *". See the note in ANTLRTokenStream.h on
* the "behavior" of ANTLRTokenStream.
*/
class ANTLRParser; // MR1
class DllExportPCCTS ANTLRTokenBuffer {
protected:
ANTLRTokenStream *input; // where do I get tokens
int buffer_size;
int chunk_size;
int num_markers;
int k; // Need at least this many tokens in buffer
_ANTLRTokenPtr *buffer; // buffer used for arbitrary lookahead
_ANTLRTokenPtr *tp; // pts into buffer; current token ptr
_ANTLRTokenPtr *last; // pts to last valid token in buffer
_ANTLRTokenPtr *next; // place to put token from getANTLRToken()
_ANTLRTokenPtr *end_of_buffer;
/* when you try to write a token past this and there are no markers
set, then move k-1 tokens back to the beginning of the buffer.
We want to stay away from the end of the buffer because we have
to extend it if a marker is set and we reach the end (we cannot
move tokens to the beginning of the buffer in this case).
*/
_ANTLRTokenPtr *threshold;
unsigned char _deleteTokens;
// This function is filled in by the subclass; it initiates fetch of input
virtual _ANTLRTokenPtr getANTLRToken() { return input->getToken(); }
void makeRoom();
void extendBuffer();
public:
ANTLRTokenBuffer(ANTLRTokenStream *in, int k=1, int chksz=50);
virtual ~ANTLRTokenBuffer();
virtual _ANTLRTokenPtr getToken();
virtual void rewind(int pos);
virtual int mark();
virtual _ANTLRTokenPtr bufferedToken(int i);
void noGarbageCollectTokens() { _deleteTokens=0; }
void garbageCollectTokens() { _deleteTokens=1; }
virtual int bufferSize() { return buffer_size; }
virtual int minTokens() { return k; }
virtual void setMinTokens(int k_new) { k = k_new; }
virtual void panic(const char *msg); /* MR20 const */
virtual int printMessage(FILE* pFile, const char* pFormat, ...); // MR23
protected: // MR1
ANTLRParser *parser; // MR1
public: // MR1
ANTLRParser *setParser(ANTLRParser *p); // MR1
ANTLRParser *getParser(); // MR1
ANTLRTokenStream *getLexer() const { // MR12
return input;} // MR12
};
#endif
|
